{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\guota\\AppData\\Local\\Continuum\\anaconda3\\envs\\gym\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import random\n",
    "from collections import deque\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from keras import backend as K\n",
    "from keras import layers, models, optimizers\n",
    "import json\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer(object):\n",
    "\n",
    "    def __init__(self, buffer_size):\n",
    "        self.buffer_size = buffer_size\n",
    "        self.num_experiences = 0\n",
    "        self.buffer = deque()\n",
    "\n",
    "    def getBatch(self, batch_size):\n",
    "        # Randomly sample batch_size examples\n",
    "        if self.num_experiences < batch_size:\n",
    "            return random.sample(self.buffer, self.num_experiences)\n",
    "        else:\n",
    "            return random.sample(self.buffer, batch_size)\n",
    "\n",
    "    def size(self):\n",
    "        return self.buffer_size\n",
    "\n",
    "    def add(self, state, action, reward, new_state, done):\n",
    "        experience = (state, action, reward, new_state, done)\n",
    "        if self.num_experiences < self.buffer_size:\n",
    "            self.buffer.append(experience)\n",
    "            self.num_experiences += 1\n",
    "        else:\n",
    "            self.buffer.popleft()\n",
    "            self.buffer.append(experience)\n",
    "\n",
    "    def count(self):\n",
    "        # if buffer is full, return buffer size\n",
    "        # otherwise, return experience counter\n",
    "        return self.num_experiences\n",
    "\n",
    "    def erase(self):\n",
    "        self.buffer = deque()\n",
    "        self.num_experiences = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OUNoise:\n",
    "    \"\"\"Ornstein-Uhlenbeck process.\"\"\"\n",
    "\n",
    "    def __init__(self, size, mu=None, theta=0.15, sigma=0.3):\n",
    "        \"\"\"Initialize parameters and noise process.\"\"\"\n",
    "        self.size = size\n",
    "        self.mu = mu if mu is not None else np.zeros(self.size)\n",
    "        self.theta = theta\n",
    "        self.sigma = sigma\n",
    "        self.state = np.ones(self.size) * self.mu\n",
    "        self.reset()\n",
    "\n",
    "    def reset(self):\n",
    "        \"\"\"Reset the internal state (= noise) to mean (mu).\"\"\"\n",
    "        self.state = self.mu\n",
    "\n",
    "    def sample(self):\n",
    "        \"\"\"Update internal state and return it as a noise sample.\"\"\"\n",
    "        x = self.state\n",
    "        dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(len(x))\n",
    "        self.state = x + dx\n",
    "        return self.state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "HIDDEN1_UNITS = 300\n",
    "HIDDEN2_UNITS = 600\n",
    "\n",
    "class ActorNetwork(object):\n",
    "    def __init__(self, sess, state_size, action_size, BATCH_SIZE, TAU, LEARNING_RATE):\n",
    "        self.sess = sess\n",
    "        self.BATCH_SIZE = BATCH_SIZE\n",
    "        self.TAU = TAU\n",
    "        self.LEARNING_RATE = LEARNING_RATE\n",
    "\n",
    "        K.set_session(sess)\n",
    "\n",
    "        #Now create the model\n",
    "        self.model , self.weights, self.state = self.create_actor_network(state_size, action_size)   \n",
    "        self.target_model, self.target_weights, self.target_state = self.create_actor_network(state_size, action_size) \n",
    "        self.action_gradient = tf.placeholder(tf.float32,[None, action_size])\n",
    "        self.params_grad = tf.gradients(self.model.output, self.weights, -self.action_gradient)\n",
    "        grads = zip(self.params_grad, self.weights)\n",
    "        self.optimize = tf.train.AdamOptimizer(LEARNING_RATE).apply_gradients(grads)\n",
    "        self.sess.run(tf.initialize_all_variables())\n",
    "\n",
    "    def train(self, states, action_grads):\n",
    "        self.sess.run(self.optimize, feed_dict={\n",
    "            self.state: states,\n",
    "            self.action_gradient: action_grads\n",
    "        })\n",
    "\n",
    "    def target_train(self):\n",
    "        actor_weights = self.model.get_weights()\n",
    "        actor_target_weights = self.target_model.get_weights()\n",
    "        for i in range(len(actor_weights)):\n",
    "            actor_target_weights[i] = self.TAU * actor_weights[i] + (1 - self.TAU)* actor_target_weights[i]\n",
    "        self.target_model.set_weights(actor_target_weights)\n",
    "\n",
    "    def create_actor_network(self, state_size,action_dim):\n",
    "        print(\"Now we build the model\")\n",
    "        S = layers.Input(shape=[state_size])   \n",
    "        h0 = layers.Dense(HIDDEN1_UNITS, activation='relu')(S)\n",
    "        h1 = layers.Dense(HIDDEN2_UNITS, activation='relu')(h0)\n",
    "        Steering = layers.Dense(action_dim,activation='tanh')(h1)#,init=lambda shape, name: RandomNormal(shape, scale=1e-4, name=name))(h1)  \n",
    "        #Acceleration = layers.Dense(1,activation='sigmoid',init=lambda shape, name: RandomNormal(shape, scale=1e-4, name=name))(h1)   \n",
    "        #Brake = layers.Dense(1,activation='sigmoid',init=lambda shape, name: RandomNormal(shape, scale=1e-4, name=name))(h1) \n",
    "        #V = merge([Steering,Acceleration,Brake],mode='concat')          \n",
    "        model = models.Model(input=S,output=Steering)\n",
    "        return model, model.trainable_weights, S\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "HIDDEN1_UNITS = 300\n",
    "HIDDEN2_UNITS = 600\n",
    "\n",
    "class CriticNetwork(object):\n",
    "    def __init__(self, sess, state_size, action_size, BATCH_SIZE, TAU, LEARNING_RATE):\n",
    "        self.sess = sess\n",
    "        self.BATCH_SIZE = BATCH_SIZE\n",
    "        self.TAU = TAU\n",
    "        self.LEARNING_RATE = LEARNING_RATE\n",
    "        self.action_size = action_size\n",
    "        \n",
    "        K.set_session(sess)\n",
    "\n",
    "        #Now create the model\n",
    "        self.model, self.action, self.state = self.create_critic_network(state_size, action_size)  \n",
    "        self.target_model, self.target_action, self.target_state = self.create_critic_network(state_size, action_size)  \n",
    "        self.action_grads = tf.gradients(self.model.output, self.action)  #GRADIENTS for policy update\n",
    "        self.sess.run(tf.initialize_all_variables())\n",
    "\n",
    "    def gradients(self, states, actions):\n",
    "        return self.sess.run(self.action_grads, feed_dict={\n",
    "            self.state: states,\n",
    "            self.action: actions\n",
    "        })[0]\n",
    "\n",
    "    def target_train(self):\n",
    "        critic_weights = self.model.get_weights()\n",
    "        critic_target_weights = self.target_model.get_weights()\n",
    "        for i in range(len(critic_weights)):\n",
    "            critic_target_weights[i] = self.TAU * critic_weights[i] + (1 - self.TAU)* critic_target_weights[i]\n",
    "        self.target_model.set_weights(critic_target_weights)\n",
    "\n",
    "    def create_critic_network(self, state_size,action_dim):\n",
    "        print(\"Now we build the model\")\n",
    "        S = layers.Input(shape=[state_size])  \n",
    "        A = layers.Input(shape=[action_dim],name='action2')   \n",
    "        w1 = layers.Dense(HIDDEN1_UNITS, activation='relu')(S)\n",
    "        a1 = layers.Dense(HIDDEN2_UNITS, activation='linear')(A) \n",
    "        h1 = layers.Dense(HIDDEN2_UNITS, activation='linear')(w1)\n",
    "        h2 = layers.add([h1,a1])\n",
    "        h3 = layers.Dense(HIDDEN2_UNITS, activation='relu')(h2)\n",
    "        V = layers.Dense(action_dim,activation='linear')(h3)   \n",
    "        model = models.Model(input=[S,A],output=V)\n",
    "        adam = optimizers.Adam(lr=self.LEARNING_RATE)\n",
    "        model.compile(loss='mse', optimizer=adam)\n",
    "        return model, A, S "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "[0.19525401]\n",
      "[0.43037874 0.20552675 0.71813095]\n",
      "[0.1]\n",
      "Now we build the model\n",
      "Now we build the model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\guota\\AppData\\Local\\Continuum\\anaconda3\\envs\\gym\\lib\\site-packages\\ipykernel_launcher.py:44: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=Tensor(\"in..., outputs=Tensor(\"de...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\guota\\AppData\\Local\\Continuum\\anaconda3\\envs\\gym\\lib\\site-packages\\tensorflow\\python\\util\\tf_should_use.py:170: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "Now we build the model\n",
      "Now we build the model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\guota\\AppData\\Local\\Continuum\\anaconda3\\envs\\gym\\lib\\site-packages\\ipykernel_launcher.py:43: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=[<tf.Tenso..., outputs=Tensor(\"de...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\guota\\AppData\\Local\\Continuum\\anaconda3\\envs\\gym\\lib\\site-packages\\tensorflow\\python\\util\\tf_should_use.py:170: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "Now we load the weight\n",
      "Cannot find the weight\n",
      "TORCS Experiment Start.\n",
      "Episode : 0 Replay Buffer 0\n",
      "[ 0.17551746  0.98447632 -0.91811875]\n",
      "Episode 0 Step 0 Action [[0.09771489]] Reward -2.0285574171212115 Loss 4.386626720428467\n",
      "Episode 0 Step 1 Action [[0.21302279]] Reward -1.9240715736497076 Loss 2.3559961318969727\n",
      "Episode 0 Step 2 Action [[0.22372944]] Reward -2.0424897875249144 Loss 1.3902406692504883\n",
      "Episode 0 Step 3 Action [[0.13087172]] Reward -2.3960740648472356 Loss 0.7595455646514893\n",
      "Episode 0 Step 4 Action [[0.15095384]] Reward -2.9965427238863227 Loss 0.2092847377061844\n",
      "Episode 0 Step 5 Action [[-0.40758734]] Reward -3.873839362612439 Loss 0.2222820520401001\n",
      "Episode 0 Step 6 Action [[-0.51896926]] Reward -4.973403183563328 Loss 0.8153438568115234\n",
      "Episode 0 Step 7 Action [[-1.00622884]] Reward -6.319559683674078 Loss 0.7269876003265381\n",
      "Episode 0 Step 8 Action [[-1.03231137]] Reward -7.8110283472098505 Loss 1.4094901084899902\n",
      "Episode 0 Step 9 Action [[-0.79984193]] Reward -9.439400589848939 Loss 1.8487077951431274\n",
      "Episode 0 Step 10 Action [[-1.03379708]] Reward -11.172880762342158 Loss 1.9698890447616577\n",
      "Episode 0 Step 11 Action [[-1.01613827]] Reward -12.675281356941456 Loss 2.6858325004577637\n",
      "Episode 0 Step 12 Action [[-0.88631132]] Reward -10.942333778248216 Loss 2.858196496963501\n",
      "Episode 0 Step 13 Action [[-0.64881398]] Reward -9.265899934822306 Loss 1.8111681938171387\n",
      "Episode 0 Step 14 Action [[-0.43510052]] Reward -7.724065093778521 Loss 1.3455902338027954\n",
      "Episode 0 Step 15 Action [[-0.15682676]] Reward -6.3693112530533105 Loss 1.412955641746521\n",
      "Episode 0 Step 16 Action [[0.0849785]] Reward -5.244580820069546 Loss 1.112067461013794\n",
      "Episode 0 Step 17 Action [[0.225182]] Reward -4.359425522905908 Loss 1.0183043479919434\n",
      "Episode 0 Step 18 Action [[0.55727293]] Reward -3.7068901481628136 Loss 1.1713861227035522\n",
      "Episode 0 Step 19 Action [[0.74107144]] Reward -3.2812652547440706 Loss 1.0219188928604126\n",
      "Episode 0 Step 20 Action [[0.21107843]] Reward -3.0544855405816898 Loss 1.0137603282928467\n",
      "Episode 0 Step 21 Action [[-0.00787124]] Reward -3.0307789349854413 Loss 1.269848346710205\n",
      "Episode 0 Step 22 Action [[0.25825768]] Reward -3.243272498362893 Loss 1.4652314186096191\n",
      "Episode 0 Step 23 Action [[0.32809516]] Reward -3.682498790613527 Loss 1.6023821830749512\n",
      "Episode 0 Step 24 Action [[0.4648818]] Reward -4.349737523049679 Loss 1.7591272592544556\n",
      "Episode 0 Step 25 Action [[0.51227296]] Reward -5.233608261591668 Loss 1.6998265981674194\n",
      "Episode 0 Step 26 Action [[0.49187484]] Reward -6.325605333484587 Loss 1.3291714191436768\n",
      "Episode 0 Step 27 Action [[0.34980593]] Reward -7.605916340068475 Loss 0.8794286251068115\n",
      "Episode 0 Step 28 Action [[0.60476008]] Reward -9.049816057154565 Loss 0.6429682970046997\n",
      "Episode 0 Step 29 Action [[0.77098112]] Reward -10.529066155807739 Loss 0.9088332653045654\n",
      "Episode 0 Step 30 Action [[0.41701348]] Reward -11.966531561773497 Loss 1.254714012145996\n",
      "Episode 0 Step 31 Action [[0.51439467]] Reward -10.550558336265693 Loss 1.0884947776794434\n",
      "Episode 0 Step 32 Action [[0.63462864]] Reward -9.133467141237986 Loss 0.8273619413375854\n",
      "Episode 0 Step 33 Action [[0.76783251]] Reward -7.789610653542949 Loss 0.8046715259552002\n",
      "Episode 0 Step 34 Action [[0.63523037]] Reward -6.5869240169243275 Loss 0.8920610547065735\n",
      "Episode 0 Step 35 Action [[0.74291269]] Reward -5.600401762940104 Loss 0.6677797436714172\n",
      "Episode 0 Step 36 Action [[0.34829886]] Reward -4.845849537976848 Loss 0.7078986167907715\n",
      "Episode 0 Step 37 Action [[0.43916443]] Reward -4.35854201047806 Loss 0.7653182148933411\n",
      "Episode 0 Step 38 Action [[0.30963717]] Reward -4.118251248349669 Loss 0.7763727903366089\n",
      "Episode 0 Step 39 Action [[0.51232068]] Reward -4.131578048047604 Loss 0.9917178153991699\n",
      "Episode 0 Step 40 Action [[0.37625638]] Reward -4.408078224427828 Loss 1.2334139347076416\n",
      "Episode 0 Step 41 Action [[0.57040775]] Reward -4.941646490667089 Loss 1.2138206958770752\n",
      "Episode 0 Step 42 Action [[0.87170043]] Reward -5.749162299646608 Loss 1.2507596015930176\n",
      "Episode 0 Step 43 Action [[1.0433456]] Reward -6.848895542253713 Loss 1.0760316848754883\n",
      "Episode 0 Step 44 Action [[1.40805102]] Reward -8.22074837372486 Loss 1.2434804439544678\n",
      "Episode 0 Step 45 Action [[1.43299034]] Reward -9.858546954891459 Loss 1.057983160018921\n",
      "Episode 0 Step 46 Action [[1.79781512]] Reward -11.656632207934809 Loss 1.4191040992736816\n",
      "Episode 0 Step 47 Action [[1.60413575]] Reward -11.336586120267404 Loss 1.6780272722244263\n",
      "Episode 0 Step 48 Action [[1.81444517]] Reward -9.983101399752641 Loss 1.266741156578064\n",
      "Episode 0 Step 49 Action [[1.7593615]] Reward -8.617081803399676 Loss 1.0581213235855103\n",
      "Episode 0 Step 50 Action [[1.59225473]] Reward -7.260000030799778 Loss 1.2834415435791016\n",
      "Episode 0 Step 51 Action [[1.85942461]] Reward -5.966922542586654 Loss 1.2747292518615723\n",
      "Episode 0 Step 52 Action [[1.88386499]] Reward -4.838424022731993 Loss 1.1436973810195923\n",
      "Episode 0 Step 53 Action [[1.81448674]] Reward -3.868462346042464 Loss 1.214116096496582\n",
      "Episode 0 Step 54 Action [[1.16042901]] Reward -3.056765989026239 Loss 1.6035189628601074\n",
      "Episode 0 Step 55 Action [[0.99742257]] Reward -2.3678334595369415 Loss 1.2197266817092896\n",
      "Episode 0 Step 56 Action [[0.96062603]] Reward -1.8440299713746116 Loss 1.2749710083007812\n",
      "Episode 0 Step 57 Action [[0.78999087]] Reward -1.4732828095134176 Loss 1.539104700088501\n",
      "Episode 0 Step 58 Action [[0.56861306]] Reward -1.2330022498469342 Loss 1.3919873237609863\n",
      "Episode 0 Step 59 Action [[1.0021807]] Reward -1.117870889490238 Loss 1.0213358402252197\n",
      "Episode 0 Step 60 Action [[1.06102891]] Reward -1.1182994054127295 Loss 1.019860029220581\n",
      "Episode 0 Step 61 Action [[0.82831944]] Reward -1.2205494555351268 Loss 0.8359996676445007\n",
      "Episode 0 Step 62 Action [[0.89707482]] Reward -1.4455084846166644 Loss 0.6685501337051392\n",
      "Episode 0 Step 63 Action [[1.07745462]] Reward -1.8053456352731927 Loss 0.6158068180084229\n",
      "Episode 0 Step 64 Action [[0.89540329]] Reward -2.3102219984121373 Loss 0.6274862289428711\n",
      "Episode 0 Step 65 Action [[0.98480998]] Reward -3.0168423783845992 Loss 0.8942388296127319\n",
      "Episode 0 Step 66 Action [[0.64241808]] Reward -3.9350756416741843 Loss 0.8963927030563354\n",
      "Episode 0 Step 67 Action [[0.73511619]] Reward -5.147183461628541 Loss 0.3529174327850342\n",
      "Episode 0 Step 68 Action [[0.67226136]] Reward -6.6139626942179985 Loss 0.8486000895500183\n",
      "Episode 0 Step 69 Action [[0.87122209]] Reward -8.330238615172822 Loss 0.9987490177154541\n",
      "Episode 0 Step 70 Action [[1.29519854]] Reward -10.177983348806743 Loss 0.747670590877533\n",
      "Episode 0 Step 71 Action [[1.41803338]] Reward -11.988235253645607 Loss 0.6995136737823486\n",
      "Episode 0 Step 72 Action [[1.41033103]] Reward -12.6918571588194 Loss 0.858864426612854\n",
      "Episode 0 Step 73 Action [[1.72949469]] Reward -10.783374538810296 Loss 1.092371940612793\n",
      "Episode 0 Step 74 Action [[1.51194564]] Reward -8.916943477628903 Loss 0.41690850257873535\n",
      "Episode 0 Step 75 Action [[1.38742683]] Reward -7.26258548875711 Loss 0.7696611881256104\n",
      "Episode 0 Step 76 Action [[1.59942524]] Reward -5.872969139317076 Loss 0.43473300337791443\n",
      "Episode 0 Step 77 Action [[1.64196662]] Reward -4.7707161786246655 Loss 0.47448283433914185\n",
      "Episode 0 Step 78 Action [[1.22856447]] Reward -4.0103129618301665 Loss 0.5407631993293762\n",
      "Episode 0 Step 79 Action [[1.09134703]] Reward -3.6085974587013903 Loss 0.43398553133010864\n",
      "Episode 0 Step 80 Action [[0.85432279]] Reward -3.5315050779596278 Loss 0.5502681136131287\n",
      "Episode 0 Step 81 Action [[0.66156847]] Reward -3.762903159622929 Loss 0.4778498113155365\n",
      "Episode 0 Step 82 Action [[0.5558266]] Reward -4.286441883470374 Loss 0.45861175656318665\n",
      "Episode 0 Step 83 Action [[0.71161807]] Reward -5.094747308238506 Loss 0.5984969735145569\n",
      "Episode 0 Step 84 Action [[0.98569539]] Reward -6.205489387663686 Loss 0.25627055764198303\n",
      "Episode 0 Step 85 Action [[0.95273897]] Reward -7.630069580785032 Loss 0.24027135968208313\n",
      "Episode 0 Step 86 Action [[0.75479822]] Reward -9.295091569586457 Loss 0.3130207657814026\n",
      "Episode 0 Step 87 Action [[0.792838]] Reward -11.089450205419807 Loss 0.3780866265296936\n",
      "Episode 0 Step 88 Action [[1.02687637]] Reward -12.162290111389579 Loss 0.2662597596645355\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 0 Step 89 Action [[0.94919229]] Reward -10.716698917939773 Loss 0.5768368244171143\n",
      "Episode 0 Step 90 Action [[0.86307931]] Reward -9.22076532218483 Loss 0.6159770488739014\n",
      "Episode 0 Step 91 Action [[0.49840115]] Reward -7.739878861229892 Loss 0.323681116104126\n",
      "Episode 0 Step 92 Action [[0.6993827]] Reward -6.32048715712555 Loss 0.5516665577888489\n",
      "Episode 0 Step 93 Action [[0.64093031]] Reward -5.094811885312083 Loss 0.3674423098564148\n",
      "Episode 0 Step 94 Action [[0.53344052]] Reward -4.064576829308709 Loss 0.26768773794174194\n",
      "Episode 0 Step 95 Action [[0.38005492]] Reward -3.2438750631945035 Loss 0.5095789432525635\n",
      "Episode 0 Step 96 Action [[0.36503559]] Reward -2.63499786335289 Loss 0.46627727150917053\n",
      "Episode 0 Step 97 Action [[0.15550228]] Reward -2.2390246935507556 Loss 0.3748840391635895\n",
      "Episode 0 Step 98 Action [[-0.14594192]] Reward -2.045201272353974 Loss 0.658794105052948\n",
      "Episode 0 Step 99 Action [[-0.24447287]] Reward -2.0668085145555004 Loss 0.6625124216079712\n",
      "Episode 0 Step 100 Action [[0.12266476]] Reward -2.323158172433551 Loss 0.2587449252605438\n",
      "Episode 0 Step 101 Action [[0.43139987]] Reward -2.8021252558791896 Loss 0.47480136156082153\n",
      "Episode 0 Step 102 Action [[0.68039984]] Reward -3.4968755289048414 Loss 0.5445371270179749\n",
      "Episode 0 Step 103 Action [[1.09901183]] Reward -4.407138062874871 Loss 0.422768771648407\n",
      "Episode 0 Step 104 Action [[1.00908228]] Reward -5.495211878753616 Loss 0.34387603402137756\n",
      "Episode 0 Step 105 Action [[1.1199191]] Reward -6.797206652221638 Loss 0.5546634793281555\n",
      "Episode 0 Step 106 Action [[1.28203738]] Reward -8.249110452434753 Loss 0.6206531524658203\n",
      "Episode 0 Step 107 Action [[1.19413399]] Reward -9.769290385318726 Loss 0.32625067234039307\n",
      "Episode 0 Step 108 Action [[1.44601747]] Reward -11.3167139778124 Loss 1.0896275043487549\n",
      "Episode 0 Step 109 Action [[1.48551098]] Reward -11.821149789828915 Loss 0.33405864238739014\n",
      "Episode 0 Step 110 Action [[1.58733134]] Reward -10.1903006855567 Loss 0.46118271350860596\n",
      "Episode 0 Step 111 Action [[1.55614115]] Reward -8.64502040376928 Loss 0.4427378475666046\n",
      "Episode 0 Step 112 Action [[1.56972768]] Reward -7.2755451739908725 Loss 0.5443207025527954\n",
      "Episode 0 Step 113 Action [[1.37053999]] Reward -6.139649842347308 Loss 0.22682559490203857\n",
      "Episode 0 Step 114 Action [[1.64112531]] Reward -5.293824631338996 Loss 0.46021661162376404\n",
      "Episode 0 Step 115 Action [[1.64968499]] Reward -4.74215467424233 Loss 0.7644206285476685\n",
      "Episode 0 Step 116 Action [[1.17269478]] Reward -4.524204223367779 Loss 0.32188642024993896\n",
      "Episode 0 Step 117 Action [[1.43558103]] Reward -4.630117228111934 Loss 0.4158196747303009\n",
      "Episode 0 Step 118 Action [[1.53210631]] Reward -5.061996190780905 Loss 0.5253972411155701\n",
      "Episode 0 Step 119 Action [[1.71641383]] Reward -5.830465807486039 Loss 0.6456747055053711\n",
      "Episode 0 Step 120 Action [[1.83367467]] Reward -6.943219587838463 Loss 0.26186078786849976\n",
      "Episode 0 Step 121 Action [[1.92435657]] Reward -8.378639959280873 Loss 0.40125101804733276\n",
      "Episode 0 Step 122 Action [[1.66159156]] Reward -10.08600326679008 Loss 0.4011101722717285\n",
      "Episode 0 Step 123 Action [[1.94114359]] Reward -11.927188808791069 Loss 0.3928282558917999\n",
      "Episode 0 Step 124 Action [[1.69932716]] Reward -11.151286101118867 Loss 0.34502267837524414\n",
      "Episode 0 Step 125 Action [[1.61407974]] Reward -9.795819725402835 Loss 0.3642457127571106\n",
      "Episode 0 Step 126 Action [[1.31237549]] Reward -8.398311535304412 Loss 0.17460204660892487\n",
      "Episode 0 Step 127 Action [[1.24005572]] Reward -7.001717739116707 Loss 0.11935703456401825\n",
      "Episode 0 Step 128 Action [[1.58445236]] Reward -5.709681519679295 Loss 0.2657650113105774\n",
      "Episode 0 Step 129 Action [[1.51564469]] Reward -4.609126444712612 Loss 0.2899048924446106\n",
      "Episode 0 Step 130 Action [[1.19351041]] Reward -3.6710821664519813 Loss 0.2871837913990021\n",
      "Episode 0 Step 131 Action [[0.94310524]] Reward -2.8895494274601425 Loss 0.1021876186132431\n",
      "Episode 0 Step 132 Action [[0.90900455]] Reward -2.278703776941519 Loss 0.38768330216407776\n",
      "Episode 0 Step 133 Action [[0.94475124]] Reward -1.841886018116832 Loss 0.20272797346115112\n",
      "Episode 0 Step 134 Action [[1.15604546]] Reward -1.5614680646215837 Loss 0.26996514201164246\n",
      "Episode 0 Step 135 Action [[0.90410656]] Reward -1.4167422788391821 Loss 0.14327621459960938\n",
      "Episode 0 Step 136 Action [[1.09544236]] Reward -1.3949466289035741 Loss 0.12497341632843018\n",
      "Episode 0 Step 137 Action [[1.1147775]] Reward -1.494972292685997 Loss 0.09133853018283844\n",
      "Episode 0 Step 138 Action [[0.88838113]] Reward -1.7200375009134237 Loss 0.31327399611473083\n",
      "Episode 0 Step 139 Action [[1.31817166]] Reward -2.1030174648701863 Loss 0.1944131702184677\n",
      "Episode 0 Step 140 Action [[1.32005482]] Reward -2.6225854700402804 Loss 0.1243857741355896\n",
      "Episode 0 Step 141 Action [[1.36210432]] Reward -3.3179865627141636 Loss 0.11688083410263062\n",
      "Episode 0 Step 142 Action [[1.65559959]] Reward -4.206450184659636 Loss 0.11564135551452637\n",
      "Episode 0 Step 143 Action [[1.48449941]] Reward -5.258406234818404 Loss 0.2742008566856384\n",
      "Episode 0 Step 144 Action [[1.39182406]] Reward -6.519844064434789 Loss 0.07524088025093079\n",
      "Episode 0 Step 145 Action [[1.43147945]] Reward -7.9585121434973445 Loss 0.14975088834762573\n",
      "Episode 0 Step 146 Action [[1.3117336]] Reward -9.496033148529987 Loss 0.10905809700489044\n",
      "Episode 0 Step 147 Action [[1.29905326]] Reward -11.08117465772922 Loss 0.1136632189154625\n",
      "Episode 0 Step 148 Action [[1.5652818]] Reward -12.360943059057142 Loss 0.12558093667030334\n",
      "Episode 0 Step 149 Action [[1.65598858]] Reward -10.653003536827175 Loss 0.20624281466007233\n",
      "Episode 0 Step 150 Action [[1.8322051]] Reward -9.018518630668868 Loss 0.09641183912754059\n",
      "Episode 0 Step 151 Action [[2.04829576]] Reward -7.533850605292703 Loss 0.1456872522830963\n",
      "Episode 0 Step 152 Action [[1.91898055]] Reward -6.285207335625386 Loss 0.15965422987937927\n",
      "Episode 0 Step 153 Action [[1.77830716]] Reward -5.352043109972799 Loss 0.10941806435585022\n",
      "Episode 0 Step 154 Action [[1.75314839]] Reward -4.760103272693401 Loss 0.12052859365940094\n",
      "Episode 0 Step 155 Action [[1.50210167]] Reward -4.513008924112457 Loss 0.13150644302368164\n",
      "Episode 0 Step 156 Action [[1.24219786]] Reward -4.607661646286837 Loss 0.0444333553314209\n",
      "Episode 0 Step 157 Action [[1.28723848]] Reward -5.020102543988074 Loss 0.13056762516498566\n",
      "Episode 0 Step 158 Action [[1.44897861]] Reward -5.748574969133796 Loss 0.06155940145254135\n",
      "Episode 0 Step 159 Action [[1.27059024]] Reward -6.798291915159009 Loss 0.12310469150543213\n",
      "Episode 0 Step 160 Action [[1.14266178]] Reward -8.108885198086387 Loss 0.06441666185855865\n",
      "Episode 0 Step 161 Action [[1.13283667]] Reward -9.619052969635183 Loss 0.08881175518035889\n",
      "Episode 0 Step 162 Action [[1.30675565]] Reward -11.266407787449742 Loss 0.10984812676906586\n",
      "Episode 0 Step 163 Action [[1.06111251]] Reward -11.218011864900994 Loss 0.04154709354043007\n",
      "Episode 0 Step 164 Action [[1.23378106]] Reward -9.896647248330638 Loss 0.14980989694595337\n",
      "Episode 0 Step 165 Action [[1.50244487]] Reward -8.57439452659604 Loss 0.07737874984741211\n",
      "Episode 0 Step 166 Action [[1.23114336]] Reward -7.30648193318703 Loss 0.07816953957080841\n",
      "Episode 0 Step 167 Action [[1.47184052]] Reward -6.087573352404896 Loss 0.06124260276556015\n",
      "Episode 0 Step 168 Action [[1.68117975]] Reward -5.021243884195518 Loss 0.062404565513134\n",
      "Episode 0 Step 169 Action [[1.43568896]] Reward -4.118852607083372 Loss 0.07769515365362167\n",
      "Episode 0 Step 170 Action [[1.18730489]] Reward -3.3558991344807847 Loss 0.06993596255779266\n",
      "Episode 0 Step 171 Action [[0.93044921]] Reward -2.7483357705217766 Loss 0.053685903549194336\n",
      "Episode 0 Step 172 Action [[0.65291234]] Reward -2.303626989687064 Loss 0.06992735713720322\n",
      "Episode 0 Step 173 Action [[0.4173279]] Reward -2.027518933532412 Loss 0.07982713729143143\n",
      "Episode 0 Step 174 Action [[0.02078814]] Reward -1.9300821932646075 Loss 0.05858013778924942\n",
      "Episode 0 Step 175 Action [[0.38854035]] Reward -2.0332017633463533 Loss 0.054616086184978485\n",
      "Episode 0 Step 176 Action [[0.88751461]] Reward -2.332837006216763 Loss 0.11121408641338348\n",
      "Episode 0 Step 177 Action [[0.892015]] Reward -2.8019808802349973 Loss 0.09234526753425598\n",
      "Episode 0 Step 178 Action [[0.95656424]] Reward -3.4681077545010277 Loss 0.09904758632183075\n",
      "Episode 0 Step 179 Action [[1.00581765]] Reward -4.341056032291482 Loss 0.14065542817115784\n",
      "Episode 0 Step 180 Action [[0.79356264]] Reward -5.42571346525314 Loss 0.0550166517496109\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 0 Step 181 Action [[1.05765737]] Reward -6.751602128141055 Loss 0.05909299477934837\n",
      "Episode 0 Step 182 Action [[0.79254531]] Reward -8.216594778845183 Loss 0.04391201585531235\n",
      "Episode 0 Step 183 Action [[0.52168712]] Reward -9.838317320263469 Loss 0.14628484845161438\n",
      "Episode 0 Step 184 Action [[0.36628392]] Reward -11.553251419721589 Loss 0.06721151620149612\n",
      "Episode 0 Step 185 Action [[0.31386525]] Reward -12.047035423368005 Loss 0.09581401944160461\n",
      "Episode 0 Step 186 Action [[0.53493282]] Reward -10.421173546941532 Loss 0.152723029255867\n",
      "Episode 0 Step 187 Action [[1.12356256]] Reward -8.800650535815286 Loss 0.05468658357858658\n",
      "Episode 0 Step 188 Action [[1.07175439]] Reward -7.238724751082437 Loss 0.11463449150323868\n",
      "Episode 0 Step 189 Action [[1.27101435]] Reward -5.905264228050759 Loss 0.05398008972406387\n",
      "Episode 0 Step 190 Action [[1.55114706]] Reward -4.833615587965967 Loss 0.07483112812042236\n",
      "Episode 0 Step 191 Action [[1.25070043]] Reward -4.068742818874999 Loss 0.05934371426701546\n",
      "Episode 0 Step 192 Action [[1.22240343]] Reward -3.6543929774830315 Loss 0.09204521030187607\n",
      "Episode 0 Step 193 Action [[1.03583667]] Reward -3.568814505767111 Loss 0.06348776817321777\n",
      "Episode 0 Step 194 Action [[1.0203545]] Reward -3.804940938963641 Loss 0.04980212450027466\n",
      "Episode 0 Step 195 Action [[0.8731681]] Reward -4.361198148039659 Loss 0.055700697004795074\n",
      "Episode 0 Step 196 Action [[1.22918135]] Reward -5.22663094159881 Loss 0.059438206255435944\n",
      "Episode 0 Step 197 Action [[1.5090349]] Reward -6.445671416370647 Loss 0.02103576809167862\n",
      "Episode 0 Step 198 Action [[1.53607517]] Reward -8.019096829628822 Loss 0.05960315465927124\n",
      "Episode 0 Step 199 Action [[1.66849124]] Reward -9.875702822981586 Loss 0.06573653221130371\n",
      "Now we save model\n",
      "TOTAL REWARD @ 0-th Episode  : Reward -1206.7666853353119\n",
      "Total Step: 200\n",
      "\n",
      "Episode : 1 Replay Buffer 200\n",
      "[ 0.96549037 -0.26043877  0.11493825]\n",
      "Episode 1 Step 200 Action [[0.6988978]] Reward -0.07122947075548582 Loss 0.04408467561006546\n",
      "Episode 1 Step 201 Action [[0.38726489]] Reward -0.06898711246934976 Loss 0.08141665160655975\n",
      "Episode 1 Step 202 Action [[0.36686692]] Reward -0.0731305803907913 Loss 0.09080127626657486\n",
      "Episode 1 Step 203 Action [[0.61902278]] Reward -0.08565162914834773 Loss 0.029844462871551514\n",
      "Episode 1 Step 204 Action [[0.6947992]] Reward -0.10366899364939683 Loss 0.05571064352989197\n",
      "Episode 1 Step 205 Action [[0.67997032]] Reward -0.12882276174537363 Loss 0.03345152735710144\n",
      "Episode 1 Step 206 Action [[0.3040511]] Reward -0.16473913888324979 Loss 0.04424304887652397\n",
      "Episode 1 Step 207 Action [[0.29495894]] Reward -0.2276240953360837 Loss 0.05083712562918663\n",
      "Episode 1 Step 208 Action [[0.74063993]] Reward -0.3211739474168432 Loss 0.0800279974937439\n",
      "Episode 1 Step 209 Action [[1.07572336]] Reward -0.43596106097500864 Loss 0.12579850852489471\n",
      "Episode 1 Step 210 Action [[1.12323022]] Reward -0.5780492108430438 Loss 0.030484812334179878\n",
      "Episode 1 Step 211 Action [[0.81755458]] Reward -0.7705186300854925 Loss 0.03283008933067322\n",
      "Episode 1 Step 212 Action [[1.25791984]] Reward -1.0600712328533408 Loss 0.04950127750635147\n",
      "Episode 1 Step 213 Action [[1.05406303]] Reward -1.4217197515683684 Loss 0.03499285876750946\n",
      "Episode 1 Step 214 Action [[0.83308667]] Reward -1.9333855669254594 Loss 0.03763066604733467\n",
      "Episode 1 Step 215 Action [[1.12818421]] Reward -2.6511112523262983 Loss 0.02255360223352909\n",
      "Episode 1 Step 216 Action [[1.12707667]] Reward -3.5641767119211054 Loss 0.09563842415809631\n",
      "Episode 1 Step 217 Action [[1.05089958]] Reward -4.7383821171729235 Loss 0.04521102085709572\n",
      "Episode 1 Step 218 Action [[0.85538504]] Reward -6.2094388780865035 Loss 0.04617765545845032\n",
      "Episode 1 Step 219 Action [[0.76923423]] Reward -7.996957900291888 Loss 0.08494864404201508\n",
      "Episode 1 Step 220 Action [[0.77839652]] Reward -10.033808160450633 Loss 0.01591237261891365\n",
      "Episode 1 Step 221 Action [[1.17336964]] Reward -12.193599321007138 Loss 0.0980033427476883\n",
      "Episode 1 Step 222 Action [[1.32846425]] Reward -14.169540787796123 Loss 0.12415503710508347\n",
      "Episode 1 Step 223 Action [[1.34865553]] Reward -12.007145207990856 Loss 0.02538224309682846\n",
      "Episode 1 Step 224 Action [[1.75540318]] Reward -9.88016558656183 Loss 0.04680202901363373\n",
      "Episode 1 Step 225 Action [[1.90221268]] Reward -7.847070004752672 Loss 0.04538162052631378\n",
      "Episode 1 Step 226 Action [[1.72659616]] Reward -6.075460182537486 Loss 0.023265477269887924\n",
      "Episode 1 Step 227 Action [[1.5918541]] Reward -4.676555462100312 Loss 0.024019096046686172\n",
      "Episode 1 Step 228 Action [[1.33623055]] Reward -3.6592795390060866 Loss 0.12943917512893677\n",
      "Episode 1 Step 229 Action [[1.05192695]] Reward -3.015221756326065 Loss 0.03677963837981224\n",
      "Episode 1 Step 230 Action [[0.39326578]] Reward -2.7094225111108305 Loss 0.043636396527290344\n",
      "Episode 1 Step 231 Action [[0.48221396]] Reward -2.6971598021105616 Loss 0.2689465880393982\n",
      "Episode 1 Step 232 Action [[0.06491614]] Reward -2.9546917801002 Loss 0.02906344272196293\n",
      "Episode 1 Step 233 Action [[0.46280292]] Reward -3.4613821436484975 Loss 0.04663439840078354\n",
      "Episode 1 Step 234 Action [[0.39385663]] Reward -4.2728233788665335 Loss 0.10016215592622757\n",
      "Episode 1 Step 235 Action [[0.40580564]] Reward -5.379371482059273 Loss 0.048298705369234085\n",
      "Episode 1 Step 236 Action [[0.65308712]] Reward -6.781707118912796 Loss 0.017044156789779663\n",
      "Episode 1 Step 237 Action [[0.83145963]] Reward -8.49042638504252 Loss 0.06098540127277374\n",
      "Episode 1 Step 238 Action [[0.62069874]] Reward -10.448088720308178 Loss 0.037263501435518265\n",
      "Episode 1 Step 239 Action [[0.63594064]] Reward -12.486796994127015 Loss 0.08280874043703079\n",
      "Episode 1 Step 240 Action [[0.23282502]] Reward -12.015682245938686 Loss 0.03593657910823822\n",
      "Episode 1 Step 241 Action [[0.1421682]] Reward -10.29359391327586 Loss 0.0568661205470562\n",
      "Episode 1 Step 242 Action [[-0.30133953]] Reward -8.580237546545128 Loss 0.035299599170684814\n",
      "Episode 1 Step 243 Action [[-0.59624788]] Reward -6.9252491353390315 Loss 0.040665194392204285\n",
      "Episode 1 Step 244 Action [[-0.88705019]] Reward -5.45299185016091 Loss 0.07134076207876205\n",
      "Episode 1 Step 245 Action [[-0.6511063]] Reward -4.2372813027411995 Loss 0.03656614571809769\n",
      "Episode 1 Step 246 Action [[-1.09525586]] Reward -3.3500717296207085 Loss 0.031037669628858566\n",
      "Episode 1 Step 247 Action [[-1.11578636]] Reward -2.7470749832487034 Loss 0.15329845249652863\n",
      "Episode 1 Step 248 Action [[-0.85944779]] Reward -2.460962639384904 Loss 0.0318654403090477\n",
      "Episode 1 Step 249 Action [[-0.41981612]] Reward -2.47866082040412 Loss 0.04579269140958786\n",
      "Episode 1 Step 250 Action [[-0.27801129]] Reward -2.7673733143764 Loss 0.1285746544599533\n",
      "Episode 1 Step 251 Action [[0.26092574]] Reward -3.320385531351709 Loss 0.046268951147794724\n",
      "Episode 1 Step 252 Action [[0.6063493]] Reward -4.098852449927038 Loss 0.05059108883142471\n",
      "Episode 1 Step 253 Action [[0.77795438]] Reward -5.087974731895496 Loss 0.10556483268737793\n",
      "Episode 1 Step 254 Action [[0.64277951]] Reward -6.2785442047145255 Loss 0.08534291386604309\n",
      "Episode 1 Step 255 Action [[0.52882725]] Reward -7.68125099466895 Loss 0.06299188733100891\n",
      "Episode 1 Step 256 Action [[0.65158823]] Reward -9.252199102562457 Loss 0.07749646157026291\n",
      "Episode 1 Step 257 Action [[0.9183303]] Reward -10.878552335539219 Loss 0.1419421285390854\n",
      "Episode 1 Step 258 Action [[0.95320403]] Reward -12.324026553224899 Loss 0.062341298907995224\n",
      "Episode 1 Step 259 Action [[0.7280339]] Reward -10.712643701796551 Loss 0.09764046221971512\n",
      "Episode 1 Step 260 Action [[0.98544758]] Reward -9.167402406010451 Loss 0.20955583453178406\n",
      "Episode 1 Step 261 Action [[0.89152878]] Reward -7.698948288127937 Loss 0.04747268557548523\n",
      "Episode 1 Step 262 Action [[0.84562995]] Reward -6.4204366467119955 Loss 0.05097579583525658\n",
      "Episode 1 Step 263 Action [[0.77259555]] Reward -5.374796205093485 Loss 0.12368335574865341\n",
      "Episode 1 Step 264 Action [[0.37723321]] Reward -4.59326316457563 Loss 0.08699125051498413\n",
      "Episode 1 Step 265 Action [[0.48680805]] Reward -4.091241456093722 Loss 0.020711347460746765\n",
      "Episode 1 Step 266 Action [[0.25507371]] Reward -3.845194927912473 Loss 0.06644069403409958\n",
      "Episode 1 Step 267 Action [[0.24090716]] Reward -3.858212239406204 Loss 0.08621940016746521\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 1 Step 268 Action [[0.1225314]] Reward -4.123415709595142 Loss 0.03804449737071991\n",
      "Episode 1 Step 269 Action [[0.1402396]] Reward -4.633435520842527 Loss 0.021244622766971588\n",
      "Episode 1 Step 270 Action [[0.1642431]] Reward -5.389227770969808 Loss 0.03864167630672455\n",
      "Episode 1 Step 271 Action [[0.31637034]] Reward -6.383047734156065 Loss 0.07687333226203918\n",
      "Episode 1 Step 272 Action [[0.48561847]] Reward -7.610362454726538 Loss 0.01681937277317047\n",
      "Episode 1 Step 273 Action [[0.49760938]] Reward -9.043036043283914 Loss 0.04416751489043236\n",
      "Episode 1 Step 274 Action [[0.39756142]] Reward -10.60310517223296 Loss 0.07684950530529022\n",
      "Episode 1 Step 275 Action [[0.59405159]] Reward -11.764048059272241 Loss 0.034112218767404556\n",
      "Episode 1 Step 276 Action [[0.54102141]] Reward -10.430649144260567 Loss 0.0642639771103859\n",
      "Episode 1 Step 277 Action [[0.52261599]] Reward -9.070534971697725 Loss 0.05555964261293411\n",
      "Episode 1 Step 278 Action [[0.66737883]] Reward -7.744262414457517 Loss 0.01916687563061714\n",
      "Episode 1 Step 279 Action [[0.53038246]] Reward -6.5237596591416604 Loss 0.057442110031843185\n",
      "Episode 1 Step 280 Action [[0.40449655]] Reward -5.429880267922894 Loss 0.03791945427656174\n",
      "Episode 1 Step 281 Action [[0.57627449]] Reward -4.509241314091016 Loss 0.02015700936317444\n",
      "Episode 1 Step 282 Action [[0.04200468]] Reward -3.8012800487009124 Loss 0.014340702444314957\n",
      "Episode 1 Step 283 Action [[-0.03796286]] Reward -3.2861496564130825 Loss 0.03016049973666668\n",
      "Episode 1 Step 284 Action [[-0.07266901]] Reward -3.007892030413714 Loss 0.0119305569678545\n",
      "Episode 1 Step 285 Action [[0.17902913]] Reward -2.9708466123446953 Loss 0.03069065883755684\n",
      "Episode 1 Step 286 Action [[0.48459488]] Reward -3.1658234834542816 Loss 0.017892392352223396\n",
      "Episode 1 Step 287 Action [[1.22051887]] Reward -3.572673253575982 Loss 0.02882084622979164\n",
      "Episode 1 Step 288 Action [[1.37344001]] Reward -4.130927945514288 Loss 0.013436894863843918\n",
      "Episode 1 Step 289 Action [[1.18571653]] Reward -4.844841443876476 Loss 0.033830009400844574\n",
      "Episode 1 Step 290 Action [[1.17472196]] Reward -5.739360889669162 Loss 0.02440340630710125\n",
      "Episode 1 Step 291 Action [[1.30512388]] Reward -6.787159225482007 Loss 0.05537357181310654\n",
      "Episode 1 Step 292 Action [[1.00313531]] Reward -7.933221188682377 Loss 0.003276144852861762\n",
      "Episode 1 Step 293 Action [[0.82736605]] Reward -9.190963408051456 Loss 0.03971179202198982\n",
      "Episode 1 Step 294 Action [[0.79381036]] Reward -10.499622123031806 Loss 0.026287872344255447\n",
      "Episode 1 Step 295 Action [[0.3721804]] Reward -11.393653612591168 Loss 0.03756145387887955\n",
      "Episode 1 Step 296 Action [[0.48539799]] Reward -10.105959038667812 Loss 0.0394781269133091\n",
      "Episode 1 Step 297 Action [[0.82241151]] Reward -8.838393469066002 Loss 0.02261439897119999\n",
      "Episode 1 Step 298 Action [[1.07294535]] Reward -7.636383507311213 Loss 0.050396110862493515\n",
      "Episode 1 Step 299 Action [[0.67658009]] Reward -6.581534610321422 Loss 0.03224854916334152\n",
      "Episode 1 Step 300 Action [[0.68733647]] Reward -5.761738367824029 Loss 0.02287028171122074\n",
      "Episode 1 Step 301 Action [[0.55944237]] Reward -5.166535714231683 Loss 0.044921875\n",
      "Episode 1 Step 302 Action [[0.7136341]] Reward -4.8157748784253425 Loss 0.005042048171162605\n",
      "Episode 1 Step 303 Action [[0.55381992]] Reward -4.715470080285102 Loss 0.01390930823981762\n",
      "Episode 1 Step 304 Action [[0.78775852]] Reward -4.871065384278206 Loss 0.014763476327061653\n",
      "Episode 1 Step 305 Action [[0.63518255]] Reward -5.2949379669318315 Loss 0.01715671271085739\n",
      "Episode 1 Step 306 Action [[0.94445945]] Reward -5.969593457049078 Loss 0.028270430862903595\n",
      "Episode 1 Step 307 Action [[0.87274805]] Reward -6.9171273469549615 Loss 0.018169477581977844\n",
      "Episode 1 Step 308 Action [[1.00350946]] Reward -8.087422929419414 Loss 0.021547727286815643\n",
      "Episode 1 Step 309 Action [[0.86755875]] Reward -9.457008538927731 Loss 0.018394319340586662\n",
      "Episode 1 Step 310 Action [[0.52660606]] Reward -10.929043746266153 Loss 0.017804935574531555\n",
      "Episode 1 Step 311 Action [[0.56233615]] Reward -10.947554270293226 Loss 0.013903875835239887\n",
      "Episode 1 Step 312 Action [[0.74888603]] Reward -9.713307755059045 Loss 0.008215874433517456\n",
      "Episode 1 Step 313 Action [[0.60796227]] Reward -8.49603518014363 Loss 0.012558992952108383\n",
      "Episode 1 Step 314 Action [[0.57819838]] Reward -7.310697822314534 Loss 0.006621738895773888\n",
      "Episode 1 Step 315 Action [[0.80878421]] Reward -6.223198619768334 Loss 0.0038083097897469997\n",
      "Episode 1 Step 316 Action [[0.78781178]] Reward -5.291250293007825 Loss 0.015079762786626816\n",
      "Episode 1 Step 317 Action [[0.55175618]] Reward -4.516548387664297 Loss 0.009421228431165218\n",
      "Episode 1 Step 318 Action [[0.52614042]] Reward -3.912085161898192 Loss 0.014323463663458824\n",
      "Episode 1 Step 319 Action [[0.39756616]] Reward -3.5051590395567187 Loss 0.007094361819326878\n",
      "Episode 1 Step 320 Action [[0.2938097]] Reward -3.3008804450883518 Loss 0.011573906987905502\n",
      "Episode 1 Step 321 Action [[0.53704326]] Reward -3.3104364551971313 Loss 0.004598158411681652\n",
      "Episode 1 Step 322 Action [[0.90667209]] Reward -3.527001046595431 Loss 0.03407000005245209\n",
      "Episode 1 Step 323 Action [[1.06515624]] Reward -3.9236928030329765 Loss 0.020257888361811638\n",
      "Episode 1 Step 324 Action [[1.53759075]] Reward -4.492400715158753 Loss 0.012278069742023945\n",
      "Episode 1 Step 325 Action [[1.59112837]] Reward -5.183639987763982 Loss 0.052697062492370605\n",
      "Episode 1 Step 326 Action [[1.34693024]] Reward -6.001224627720737 Loss 0.015263638459146023\n",
      "Episode 1 Step 327 Action [[1.15963111]] Reward -6.960363900728423 Loss 0.01095869392156601\n",
      "Episode 1 Step 328 Action [[1.34506724]] Reward -8.03788459368448 Loss 0.014071333222091198\n",
      "Episode 1 Step 329 Action [[1.47319606]] Reward -9.143896413067429 Loss 0.01090934220701456\n",
      "Episode 1 Step 330 Action [[1.49490924]] Reward -10.224787306497653 Loss 0.03296387195587158\n",
      "Episode 1 Step 331 Action [[1.23875363]] Reward -11.031775888674101 Loss 0.04782217741012573\n",
      "Episode 1 Step 332 Action [[0.95025212]] Reward -9.87281873373612 Loss 0.012239960953593254\n",
      "Episode 1 Step 333 Action [[0.72093999]] Reward -8.805623945430243 Loss 0.05829586088657379\n",
      "Episode 1 Step 334 Action [[0.77932284]] Reward -7.854413766724394 Loss 0.10042303055524826\n",
      "Episode 1 Step 335 Action [[0.68152075]] Reward -7.034981846048924 Loss 0.009530773386359215\n",
      "Episode 1 Step 336 Action [[0.9589388]] Reward -6.390351194357988 Loss 0.030577093362808228\n",
      "Episode 1 Step 337 Action [[1.00695996]] Reward -5.938632982264811 Loss 0.06167009472846985\n",
      "Episode 1 Step 338 Action [[0.96228963]] Reward -5.718680061116177 Loss 0.038106393069028854\n",
      "Episode 1 Step 339 Action [[0.79623915]] Reward -5.741527469201011 Loss 0.005980983376502991\n",
      "Episode 1 Step 340 Action [[1.03409234]] Reward -5.999183421882906 Loss 0.05580876022577286\n",
      "Episode 1 Step 341 Action [[0.85731377]] Reward -6.502024618471177 Loss 0.027962302789092064\n",
      "Episode 1 Step 342 Action [[0.67702031]] Reward -7.217895799527178 Loss 0.020961079746484756\n",
      "Episode 1 Step 343 Action [[1.01187648]] Reward -8.10581802953698 Loss 0.030559901148080826\n",
      "Episode 1 Step 344 Action [[1.2835868]] Reward -9.18060479629968 Loss 0.017175335437059402\n",
      "Episode 1 Step 345 Action [[1.33685119]] Reward -10.412777840824376 Loss 0.03676369786262512\n",
      "Episode 1 Step 346 Action [[1.27950301]] Reward -10.62841993009486 Loss 0.010807323269546032\n",
      "Episode 1 Step 347 Action [[1.06444747]] Reward -9.61515443286909 Loss 0.009337017312645912\n",
      "Episode 1 Step 348 Action [[0.9229743]] Reward -8.558522569971469 Loss 0.04740750044584274\n",
      "Episode 1 Step 349 Action [[0.84706971]] Reward -7.511739079137974 Loss 0.016301050782203674\n",
      "Episode 1 Step 350 Action [[1.05306584]] Reward -6.528466720685972 Loss 0.010948240756988525\n",
      "Episode 1 Step 351 Action [[0.81643636]] Reward -5.664256167446511 Loss 0.02269294485449791\n",
      "Episode 1 Step 352 Action [[0.95203329]] Reward -4.918917162720306 Loss 0.010032618418335915\n",
      "Episode 1 Step 353 Action [[0.76144849]] Reward -4.337006695326597 Loss 0.025702498853206635\n",
      "Episode 1 Step 354 Action [[1.05277516]] Reward -3.917839830920924 Loss 0.013134552165865898\n",
      "Episode 1 Step 355 Action [[0.8720689]] Reward -3.675879379498046 Loss 0.0254667978733778\n",
      "Episode 1 Step 356 Action [[0.89225038]] Reward -3.603263546684329 Loss 0.02546672709286213\n",
      "Episode 1 Step 357 Action [[1.00666998]] Reward -3.707433665341438 Loss 0.01017867960035801\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 1 Step 358 Action [[1.25240725]] Reward -3.9831265055070126 Loss 0.012086416594684124\n",
      "Episode 1 Step 359 Action [[0.91769108]] Reward -4.410673676461174 Loss 0.01745748147368431\n",
      "Episode 1 Step 360 Action [[1.07293027]] Reward -5.024837206012746 Loss 0.012518897652626038\n",
      "Episode 1 Step 361 Action [[1.22315952]] Reward -5.7904035887725565 Loss 0.004708573687821627\n",
      "Episode 1 Step 362 Action [[0.9629047]] Reward -6.677924839323314 Loss 0.019893759861588478\n",
      "Episode 1 Step 363 Action [[0.84644382]] Reward -7.704709314300561 Loss 0.01529740821570158\n",
      "Episode 1 Step 364 Action [[1.50301694]] Reward -8.829083096053425 Loss 0.03699271008372307\n",
      "Episode 1 Step 365 Action [[1.64718391]] Reward -9.887127955206177 Loss 0.01864212192595005\n",
      "Episode 1 Step 366 Action [[1.72708344]] Reward -10.874394384295293 Loss 0.007058494258671999\n",
      "Episode 1 Step 367 Action [[1.806645]] Reward -10.136817555293948 Loss 0.030539901927113533\n",
      "Episode 1 Step 368 Action [[1.99113056]] Reward -9.064799399953959 Loss 0.018960248678922653\n",
      "Episode 1 Step 369 Action [[1.74959602]] Reward -8.131377253609152 Loss 0.02151017263531685\n",
      "Episode 1 Step 370 Action [[1.63906438]] Reward -7.404243678542964 Loss 0.005486093461513519\n",
      "Episode 1 Step 371 Action [[1.3452778]] Reward -6.895593311161469 Loss 0.006857952568680048\n",
      "Episode 1 Step 372 Action [[1.05657794]] Reward -6.6165512764182015 Loss 0.017922837287187576\n",
      "Episode 1 Step 373 Action [[1.10411945]] Reward -6.5551353291477366 Loss 0.01204475574195385\n",
      "Episode 1 Step 374 Action [[0.75054232]] Reward -6.706966260924263 Loss 0.012643037363886833\n",
      "Episode 1 Step 375 Action [[0.72429207]] Reward -7.046522045190454 Loss 0.013630229979753494\n",
      "Episode 1 Step 376 Action [[0.52033491]] Reward -7.559216793405162 Loss 0.009798211045563221\n",
      "Episode 1 Step 377 Action [[0.24621356]] Reward -8.206484260469967 Loss 0.007165876217186451\n",
      "Episode 1 Step 378 Action [[0.1112268]] Reward -8.9383640818118 Loss 0.02577913925051689\n",
      "Episode 1 Step 379 Action [[0.08604644]] Reward -9.719981315042734 Loss 0.009917658753693104\n",
      "Episode 1 Step 380 Action [[-0.07377559]] Reward -10.412440572397998 Loss 0.018155377358198166\n",
      "Episode 1 Step 381 Action [[-0.09088605]] Reward -9.6582904396962 Loss 0.018544582650065422\n",
      "Episode 1 Step 382 Action [[0.04668069]] Reward -8.921640049195167 Loss 0.024594765156507492\n",
      "Episode 1 Step 383 Action [[0.14271542]] Reward -8.235080702272066 Loss 0.008571146056056023\n",
      "Episode 1 Step 384 Action [[0.24610624]] Reward -7.6174466290763325 Loss 0.021161504089832306\n",
      "Episode 1 Step 385 Action [[0.27295098]] Reward -7.087775059477771 Loss 0.014464091509580612\n",
      "Episode 1 Step 386 Action [[0.42797441]] Reward -6.6606634144516965 Loss 0.037703707814216614\n",
      "Episode 1 Step 387 Action [[0.21402661]] Reward -6.3494677543216484 Loss 0.010890774428844452\n",
      "Episode 1 Step 388 Action [[0.59287427]] Reward -6.164036665856551 Loss 0.021799447014927864\n",
      "Episode 1 Step 389 Action [[0.86395807]] Reward -6.107272498780468 Loss 0.024729637429118156\n",
      "Episode 1 Step 390 Action [[0.80742046]] Reward -6.159594314752994 Loss 0.011011829599738121\n",
      "Episode 1 Step 391 Action [[0.77952755]] Reward -6.319873295593819 Loss 0.015438123606145382\n",
      "Episode 1 Step 392 Action [[0.67379333]] Reward -6.5856037390244735 Loss 0.010752489790320396\n",
      "Episode 1 Step 393 Action [[0.57026723]] Reward -6.956290829909896 Loss 0.020844992250204086\n",
      "Episode 1 Step 394 Action [[1.19049279]] Reward -7.4274122569720324 Loss 0.01681452989578247\n",
      "Episode 1 Step 395 Action [[1.3958162]] Reward -7.920006390244911 Loss 0.019417712464928627\n",
      "Episode 1 Step 396 Action [[1.24405806]] Reward -8.41811876813512 Loss 0.01086562778800726\n",
      "Episode 1 Step 397 Action [[1.08858423]] Reward -8.926790678637484 Loss 0.018935397267341614\n",
      "Episode 1 Step 398 Action [[1.12353979]] Reward -9.433914553242108 Loss 0.02556113339960575\n",
      "Episode 1 Step 399 Action [[1.08218121]] Reward -9.9078682118869 Loss 0.006363212130963802\n",
      "TOTAL REWARD @ 1-th Episode  : Reward -1274.4980053325853\n",
      "Total Step: 400\n",
      "\n",
      "Episode : 2 Replay Buffer 400\n",
      "[-0.81040949  0.58586386  0.45077511]\n",
      "Episode 2 Step 400 Action [[0.4437114]] Reward -6.348996770679797 Loss 0.030188996344804764\n",
      "Episode 2 Step 401 Action [[0.45517822]] Reward -6.663188192863704 Loss 0.014243530109524727\n",
      "Episode 2 Step 402 Action [[0.69666823]] Reward -7.150774132541161 Loss 0.014614004641771317\n",
      "Episode 2 Step 403 Action [[0.98333543]] Reward -7.818116759019973 Loss 0.01797318272292614\n",
      "Episode 2 Step 404 Action [[0.68206035]] Reward -8.663423305610626 Loss 0.0061407326720654964\n",
      "Episode 2 Step 405 Action [[0.58267988]] Reward -9.601809695449006 Loss 0.019200114533305168\n",
      "Episode 2 Step 406 Action [[0.74265656]] Reward -10.59525170536068 Loss 0.013448922894895077\n",
      "Episode 2 Step 407 Action [[0.65014943]] Reward -9.948361578395469 Loss 0.006583927199244499\n",
      "Episode 2 Step 408 Action [[0.62360034]] Reward -9.067538973030242 Loss 0.0069366456009447575\n",
      "Episode 2 Step 409 Action [[0.4757492]] Reward -8.196133668554591 Loss 0.014182742685079575\n",
      "Episode 2 Step 410 Action [[0.44711074]] Reward -7.364107020857351 Loss 0.006395759992301464\n",
      "Episode 2 Step 411 Action [[0.30096174]] Reward -6.616676887986946 Loss 0.016656026244163513\n",
      "Episode 2 Step 412 Action [[0.04651756]] Reward -5.982172746228818 Loss 0.011191359721124172\n",
      "Episode 2 Step 413 Action [[-0.54346141]] Reward -5.493836815970641 Loss 0.019465899094939232\n",
      "Episode 2 Step 414 Action [[-0.8347542]] Reward -5.192156113532986 Loss 0.012529618106782436\n",
      "Episode 2 Step 415 Action [[-0.82158709]] Reward -5.131176343730346 Loss 0.014608661644160748\n",
      "Episode 2 Step 416 Action [[-0.48955782]] Reward -5.327446396892024 Loss 0.00896406453102827\n",
      "Episode 2 Step 417 Action [[0.34729001]] Reward -5.755238967816533 Loss 0.0069928294979035854\n",
      "Episode 2 Step 418 Action [[0.67197152]] Reward -6.328941290645944 Loss 0.024838067591190338\n",
      "Episode 2 Step 419 Action [[0.81966923]] Reward -7.016068244535258 Loss 0.01416680496186018\n",
      "Episode 2 Step 420 Action [[1.04494542]] Reward -7.788951791107347 Loss 0.0059313965030014515\n",
      "Episode 2 Step 421 Action [[0.83896811]] Reward -8.596494431802295 Loss 0.015774792060256004\n",
      "Episode 2 Step 422 Action [[0.70231372]] Reward -9.446002053398013 Loss 0.01127061527222395\n",
      "Episode 2 Step 423 Action [[0.69104372]] Reward -10.30338925586672 Loss 0.026603562757372856\n",
      "Episode 2 Step 424 Action [[0.67817766]] Reward -10.096777349410157 Loss 0.008080244064331055\n",
      "Episode 2 Step 425 Action [[0.3703254]] Reward -9.248366474169611 Loss 0.025395771488547325\n",
      "Episode 2 Step 426 Action [[0.31395245]] Reward -8.474540239111109 Loss 0.012479572556912899\n",
      "Episode 2 Step 427 Action [[0.28310023]] Reward -7.782907414181265 Loss 0.008160660043358803\n",
      "Episode 2 Step 428 Action [[0.3213491]] Reward -7.1983810478562384 Loss 0.028587831184267998\n",
      "Episode 2 Step 429 Action [[0.19421949]] Reward -6.743101830364992 Loss 0.01527150347828865\n",
      "Episode 2 Step 430 Action [[0.39243233]] Reward -6.437069557349876 Loss 0.027282020077109337\n",
      "Episode 2 Step 431 Action [[0.23478232]] Reward -6.29163373356014 Loss 0.028561502695083618\n",
      "Episode 2 Step 432 Action [[0.28502204]] Reward -6.315445600870142 Loss 0.01281886175274849\n",
      "Episode 2 Step 433 Action [[0.38621182]] Reward -6.508508107787016 Loss 0.02012247033417225\n",
      "Episode 2 Step 434 Action [[0.31951058]] Reward -6.873572516156534 Loss 0.018776141107082367\n",
      "Episode 2 Step 435 Action [[0.20667705]] Reward -7.392883469106972 Loss 0.011202971450984478\n",
      "Episode 2 Step 436 Action [[0.41890023]] Reward -8.037689103492523 Loss 0.03681323677301407\n",
      "Episode 2 Step 437 Action [[0.69124372]] Reward -8.809080925591031 Loss 0.026711948215961456\n",
      "Episode 2 Step 438 Action [[0.42494869]] Reward -9.696311195519709 Loss 0.02638595923781395\n",
      "Episode 2 Step 439 Action [[0.35514914]] Reward -10.582678729350496 Loss 0.02490822970867157\n",
      "Episode 2 Step 440 Action [[0.2995703]] Reward -9.766894297787445 Loss 0.007140561938285828\n",
      "Episode 2 Step 441 Action [[0.38537543]] Reward -8.945593360375039 Loss 0.0150133166462183\n",
      "Episode 2 Step 442 Action [[0.67762693]] Reward -8.15678701722954 Loss 0.020710330456495285\n",
      "Episode 2 Step 443 Action [[0.83563296]] Reward -7.432652444152078 Loss 0.007621329743415117\n",
      "Episode 2 Step 444 Action [[0.44667588]] Reward -6.781095180173037 Loss 0.019957851618528366\n",
      "Episode 2 Step 445 Action [[0.46271267]] Reward -6.209470286023408 Loss 0.017862223088741302\n",
      "Episode 2 Step 446 Action [[0.31218292]] Reward -5.766177866904099 Loss 0.005451137665659189\n",
      "Episode 2 Step 447 Action [[-0.0246771]] Reward -5.467298788211381 Loss 0.016071300953626633\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 2 Step 448 Action [[-0.05011005]] Reward -5.339734124355322 Loss 0.006812188774347305\n",
      "Episode 2 Step 449 Action [[0.11686941]] Reward -5.403769877843041 Loss 0.010751241818070412\n",
      "Episode 2 Step 450 Action [[0.59718957]] Reward -5.651044507238913 Loss 0.030461464077234268\n",
      "Episode 2 Step 451 Action [[0.85050852]] Reward -6.039251789339819 Loss 0.005219322629272938\n",
      "Episode 2 Step 452 Action [[0.95225972]] Reward -6.540494307548045 Loss 0.01940462924540043\n",
      "Episode 2 Step 453 Action [[0.8905531]] Reward -7.135199062559795 Loss 0.03504488617181778\n",
      "Episode 2 Step 454 Action [[1.29247332]] Reward -7.813512612769869 Loss 0.006170140579342842\n",
      "Episode 2 Step 455 Action [[1.36040331]] Reward -8.499029450071195 Loss 0.03933338075876236\n",
      "Episode 2 Step 456 Action [[1.67218175]] Reward -9.180718745199744 Loss 0.018421625718474388\n",
      "Episode 2 Step 457 Action [[1.68810384]] Reward -9.79826768244428 Loss 0.038132280111312866\n",
      "Episode 2 Step 458 Action [[1.78787745]] Reward -10.219615141307171 Loss 0.02703556977212429\n",
      "Episode 2 Step 459 Action [[1.83877894]] Reward -9.574327151608028 Loss 0.032633304595947266\n",
      "Episode 2 Step 460 Action [[1.863307]] Reward -9.041952388906465 Loss 0.02594807744026184\n",
      "Episode 2 Step 461 Action [[1.69313268]] Reward -8.649644497926758 Loss 0.010652584955096245\n",
      "Episode 2 Step 462 Action [[1.59358533]] Reward -8.41412374546232 Loss 0.02927548810839653\n",
      "Episode 2 Step 463 Action [[1.51950588]] Reward -8.33635215283727 Loss 0.014522981829941273\n",
      "Episode 2 Step 464 Action [[1.19903566]] Reward -8.414030031331407 Loss 0.004419127944856882\n",
      "Episode 2 Step 465 Action [[1.45445441]] Reward -8.626088186650547 Loss 0.02135665901005268\n",
      "Episode 2 Step 466 Action [[1.22287743]] Reward -8.979305860372664 Loss 0.02391810342669487\n",
      "Episode 2 Step 467 Action [[0.92956715]] Reward -9.439040834230177 Loss 0.009427749551832676\n",
      "Episode 2 Step 468 Action [[0.7803611]] Reward -9.963433281588062 Loss 0.04365968704223633\n",
      "Episode 2 Step 469 Action [[0.64039735]] Reward -9.762429929953694 Loss 0.007634881883859634\n",
      "Episode 2 Step 470 Action [[0.42669327]] Reward -9.264687902856947 Loss 0.017220303416252136\n",
      "Episode 2 Step 471 Action [[0.45821708]] Reward -8.762741105286322 Loss 0.021292157471179962\n",
      "Episode 2 Step 472 Action [[0.51215798]] Reward -8.285653254492177 Loss 0.01532195508480072\n",
      "Episode 2 Step 473 Action [[0.5861083]] Reward -7.8499769960269905 Loss 0.03514327108860016\n",
      "Episode 2 Step 474 Action [[0.89793786]] Reward -7.469892101431209 Loss 0.005935841239988804\n",
      "Episode 2 Step 475 Action [[0.86250951]] Reward -7.152818182002174 Loss 0.02677283063530922\n",
      "Episode 2 Step 476 Action [[1.10931642]] Reward -6.899435388384159 Loss 0.007380791939795017\n",
      "Episode 2 Step 477 Action [[0.92152033]] Reward -6.714004244204973 Loss 0.018285434693098068\n",
      "Episode 2 Step 478 Action [[0.86410446]] Reward -6.602339000097662 Loss 0.009326587431132793\n",
      "Episode 2 Step 479 Action [[0.67042907]] Reward -6.575222857866394 Loss 0.008167184889316559\n",
      "Episode 2 Step 480 Action [[0.28555628]] Reward -6.644291613254932 Loss 0.012432770803570747\n",
      "Episode 2 Step 481 Action [[0.31507713]] Reward -6.832158340494378 Loss 0.00810335110872984\n",
      "Episode 2 Step 482 Action [[0.34225369]] Reward -7.131697316277124 Loss 0.006839722394943237\n",
      "Episode 2 Step 483 Action [[0.54436547]] Reward -7.530785025018159 Loss 0.009583339095115662\n",
      "Episode 2 Step 484 Action [[0.55545777]] Reward -7.997195713076941 Loss 0.009474353864789009\n",
      "Episode 2 Step 485 Action [[0.56529225]] Reward -8.517492481976136 Loss 0.004808397963643074\n",
      "Episode 2 Step 486 Action [[0.73194369]] Reward -9.07018583978895 Loss 0.009625820443034172\n",
      "Episode 2 Step 487 Action [[0.43939782]] Reward -9.615654458995277 Loss 0.007881790399551392\n",
      "Episode 2 Step 488 Action [[0.25189087]] Reward -10.17041494818299 Loss 0.010079201310873032\n",
      "Episode 2 Step 489 Action [[0.35113588]] Reward -9.653043437435047 Loss 0.008465911261737347\n",
      "Episode 2 Step 490 Action [[0.23922868]] Reward -9.119547104612801 Loss 0.010208929888904095\n",
      "Episode 2 Step 491 Action [[0.45106937]] Reward -8.635539751245618 Loss 0.011621139943599701\n",
      "Episode 2 Step 492 Action [[0.69483829]] Reward -8.215550720000506 Loss 0.008113866671919823\n",
      "Episode 2 Step 493 Action [[0.55889458]] Reward -7.890693433883883 Loss 0.02088846080005169\n",
      "Episode 2 Step 494 Action [[0.39260103]] Reward -7.6835143398031 Loss 0.006733867805451155\n",
      "Episode 2 Step 495 Action [[0.33250323]] Reward -7.5929747637615135 Loss 0.005733037367463112\n",
      "Episode 2 Step 496 Action [[0.43157092]] Reward -7.6167138345965 Loss 0.010623516514897346\n",
      "Episode 2 Step 497 Action [[0.5523344]] Reward -7.7592035455549055 Loss 0.00765927042812109\n",
      "Episode 2 Step 498 Action [[0.23951689]] Reward -8.023989637012257 Loss 0.005030348896980286\n",
      "Episode 2 Step 499 Action [[0.59811884]] Reward -8.377982627650173 Loss 0.010997561737895012\n",
      "Episode 2 Step 500 Action [[0.98054228]] Reward -8.841296074983969 Loss 0.009534796699881554\n",
      "Episode 2 Step 501 Action [[1.12916029]] Reward -9.42179687765438 Loss 0.017569180577993393\n",
      "Episode 2 Step 502 Action [[0.94132841]] Reward -10.09770094810654 Loss 0.014138620346784592\n",
      "Episode 2 Step 503 Action [[0.84319647]] Reward -9.790219375732786 Loss 0.013680359348654747\n",
      "Episode 2 Step 504 Action [[0.6119302]] Reward -9.1776601328104 Loss 0.00789404846727848\n",
      "Episode 2 Step 505 Action [[0.89540713]] Reward -8.554521645616378 Loss 0.02289419062435627\n",
      "Episode 2 Step 506 Action [[0.82063949]] Reward -7.961913900466083 Loss 0.005790546536445618\n",
      "Episode 2 Step 507 Action [[0.98600339]] Reward -7.4036567137725235 Loss 0.027044568210840225\n",
      "Episode 2 Step 508 Action [[1.1208007]] Reward -6.906749533030017 Loss 0.011216051876544952\n",
      "Episode 2 Step 509 Action [[0.84718524]] Reward -6.4795080751215615 Loss 0.01242571510374546\n",
      "Episode 2 Step 510 Action [[0.83890146]] Reward -6.132215004276822 Loss 0.007804389111697674\n",
      "Episode 2 Step 511 Action [[0.6146229]] Reward -5.890208105502972 Loss 0.009547661058604717\n",
      "Episode 2 Step 512 Action [[0.53961641]] Reward -5.768829998128941 Loss 0.008393301628530025\n",
      "Episode 2 Step 513 Action [[0.41673505]] Reward -5.783170971228585 Loss 0.010810567066073418\n",
      "Episode 2 Step 514 Action [[0.68664425]] Reward -5.942809084215035 Loss 0.017095711082220078\n",
      "Episode 2 Step 515 Action [[0.41952341]] Reward -6.226908956345596 Loss 0.013211259618401527\n",
      "Episode 2 Step 516 Action [[0.6652075]] Reward -6.6502490562076835 Loss 0.014787362888455391\n",
      "Episode 2 Step 517 Action [[0.80376722]] Reward -7.172493999308678 Loss 0.005686874035745859\n",
      "Episode 2 Step 518 Action [[0.82890838]] Reward -7.768090329602166 Loss 0.02542123571038246\n",
      "Episode 2 Step 519 Action [[1.06098904]] Reward -8.417219961929277 Loss 0.006673875730484724\n",
      "Episode 2 Step 520 Action [[0.89496173]] Reward -9.0680023007993 Loss 0.014641856774687767\n",
      "Episode 2 Step 521 Action [[0.74666677]] Reward -9.72615412380257 Loss 0.021876730024814606\n",
      "Episode 2 Step 522 Action [[0.8690941]] Reward -10.281998443914727 Loss 0.009135296568274498\n",
      "Episode 2 Step 523 Action [[0.95435786]] Reward -9.60898463169636 Loss 0.007855704985558987\n",
      "Episode 2 Step 524 Action [[1.43479191]] Reward -8.994400496008579 Loss 0.003599514253437519\n",
      "Episode 2 Step 525 Action [[1.13055973]] Reward -8.466233481108711 Loss 0.00519477017223835\n",
      "Episode 2 Step 526 Action [[1.28258243]] Reward -8.077403635490057 Loss 0.006446899846196175\n",
      "Episode 2 Step 527 Action [[1.15508371]] Reward -7.831449886741638 Loss 0.006866382900625467\n",
      "Episode 2 Step 528 Action [[1.32515971]] Reward -7.743966313410894 Loss 0.0072503965348005295\n",
      "Episode 2 Step 529 Action [[1.26599603]] Reward -7.823667890243618 Loss 0.00640510767698288\n",
      "Episode 2 Step 530 Action [[1.18153617]] Reward -8.06915762433159 Loss 0.0032634418457746506\n",
      "Episode 2 Step 531 Action [[1.38803759]] Reward -8.465617387005478 Loss 0.013720556162297726\n",
      "Episode 2 Step 532 Action [[1.4666192]] Reward -9.015374733170605 Loss 0.013749382458627224\n",
      "Episode 2 Step 533 Action [[1.48624039]] Reward -9.698939965558315 Loss 0.0098456721752882\n",
      "Episode 2 Step 534 Action [[0.98756087]] Reward -10.189762146910487 Loss 0.005457479506731033\n",
      "Episode 2 Step 535 Action [[1.05112183]] Reward -9.547143528949276 Loss 0.005437768064439297\n",
      "Episode 2 Step 536 Action [[0.86598122]] Reward -8.88170486120473 Loss 0.013229511678218842\n",
      "Episode 2 Step 537 Action [[0.81371505]] Reward -8.207703096188066 Loss 0.00407975260168314\n",
      "Episode 2 Step 538 Action [[0.49476962]] Reward -7.56080687833295 Loss 0.011149495840072632\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 2 Step 539 Action [[0.47850348]] Reward -6.964134607147884 Loss 0.006899228785187006\n",
      "Episode 2 Step 540 Action [[0.40961646]] Reward -6.462360456629234 Loss 0.009164171293377876\n",
      "Episode 2 Step 541 Action [[0.58291153]] Reward -6.0759750757315585 Loss 0.024487756192684174\n",
      "Episode 2 Step 542 Action [[0.54552492]] Reward -5.820320270326432 Loss 0.008204019628465176\n",
      "Episode 2 Step 543 Action [[0.39035274]] Reward -5.69791964652884 Loss 0.022954020649194717\n",
      "Episode 2 Step 544 Action [[0.42776509]] Reward -5.7215985933172755 Loss 0.01815098151564598\n",
      "Episode 2 Step 545 Action [[0.65776371]] Reward -5.8938749937149835 Loss 0.012627949938178062\n",
      "Episode 2 Step 546 Action [[0.83791474]] Reward -6.195168780206658 Loss 0.03242761641740799\n",
      "Episode 2 Step 547 Action [[1.35482983]] Reward -6.604909134850151 Loss 0.01516412291675806\n",
      "Episode 2 Step 548 Action [[1.58664706]] Reward -7.0670223788905036 Loss 0.021309593692421913\n",
      "Episode 2 Step 549 Action [[1.26999682]] Reward -7.558475379646061 Loss 0.06552566587924957\n",
      "Episode 2 Step 550 Action [[1.108509]] Reward -8.102339983521452 Loss 0.0140959732234478\n",
      "Episode 2 Step 551 Action [[0.88768713]] Reward -8.678250664834179 Loss 0.11698617041110992\n",
      "Episode 2 Step 552 Action [[0.83407806]] Reward -9.277977582901356 Loss 0.012097449973225594\n",
      "Episode 2 Step 553 Action [[0.98911974]] Reward -9.870818080371528 Loss 0.09579437971115112\n",
      "Episode 2 Step 554 Action [[0.93085788]] Reward -10.021831316755113 Loss 0.010105831548571587\n",
      "Episode 2 Step 555 Action [[1.20192997]] Reward -9.443617370430136 Loss 0.04869386553764343\n",
      "Episode 2 Step 556 Action [[1.11456408]] Reward -8.932200830357347 Loss 0.02229207754135132\n",
      "Episode 2 Step 557 Action [[1.47909811]] Reward -8.52604391363349 Loss 0.02902696654200554\n",
      "Episode 2 Step 558 Action [[1.27341775]] Reward -8.243695104366902 Loss 0.02154402621090412\n",
      "Episode 2 Step 559 Action [[0.94045211]] Reward -8.109424530363878 Loss 0.011029419489204884\n",
      "Episode 2 Step 560 Action [[0.8717792]] Reward -8.109666804063652 Loss 0.018990956246852875\n",
      "Episode 2 Step 561 Action [[0.83848953]] Reward -8.234450435089018 Loss 0.045642152428627014\n",
      "Episode 2 Step 562 Action [[0.6295522]] Reward -8.476524401118004 Loss 0.013965651392936707\n",
      "Episode 2 Step 563 Action [[0.84882563]] Reward -8.811937448760885 Loss 0.03515738993883133\n",
      "Episode 2 Step 564 Action [[0.73244455]] Reward -9.246516219866747 Loss 0.010806666687130928\n",
      "Episode 2 Step 565 Action [[0.92077939]] Reward -9.74795860249521 Loss 0.04814356565475464\n",
      "Episode 2 Step 566 Action [[0.8205213]] Reward -9.96338672261141 Loss 0.019733339548110962\n",
      "Episode 2 Step 567 Action [[0.98783832]] Reward -9.463770277391077 Loss 0.007668197154998779\n",
      "Episode 2 Step 568 Action [[1.14088031]] Reward -8.948599302915655 Loss 0.03263464570045471\n",
      "Episode 2 Step 569 Action [[0.94528606]] Reward -8.428378259934005 Loss 0.013909552246332169\n",
      "Episode 2 Step 570 Action [[1.00761617]] Reward -7.9123963562350115 Loss 0.025049686431884766\n",
      "Episode 2 Step 571 Action [[1.05957171]] Reward -7.430261025332123 Loss 0.014270180836319923\n",
      "Episode 2 Step 572 Action [[0.78070483]] Reward -6.996040020041556 Loss 0.010485260747373104\n",
      "Episode 2 Step 573 Action [[0.44265169]] Reward -6.624389683397558 Loss 0.05743175745010376\n",
      "Episode 2 Step 574 Action [[0.75223736]] Reward -6.34774333194777 Loss 0.008111555129289627\n",
      "Episode 2 Step 575 Action [[0.64132773]] Reward -6.183190233957931 Loss 0.037571996450424194\n",
      "Episode 2 Step 576 Action [[0.71015491]] Reward -6.130968830374055 Loss 0.016494672745466232\n",
      "Episode 2 Step 577 Action [[0.58522173]] Reward -6.1931133444511035 Loss 0.018864667043089867\n",
      "Episode 2 Step 578 Action [[0.61011966]] Reward -6.375053585198371 Loss 0.011456295847892761\n",
      "Episode 2 Step 579 Action [[0.74141979]] Reward -6.670556347741856 Loss 0.01376054435968399\n",
      "Episode 2 Step 580 Action [[0.9350741]] Reward -7.060240901864488 Loss 0.028947431594133377\n",
      "Episode 2 Step 581 Action [[1.10509703]] Reward -7.516191761654905 Loss 0.04282543063163757\n",
      "Episode 2 Step 582 Action [[1.25629082]] Reward -8.011360335298948 Loss 0.009168518707156181\n",
      "Episode 2 Step 583 Action [[1.20759219]] Reward -8.517987925289384 Loss 0.03741432726383209\n",
      "Episode 2 Step 584 Action [[1.04508699]] Reward -9.027624852475862 Loss 0.03307577222585678\n",
      "Episode 2 Step 585 Action [[1.30121654]] Reward -9.53464285125016 Loss 0.015511244535446167\n",
      "Episode 2 Step 586 Action [[1.16690291]] Reward -9.984808990860818 Loss 0.037529028952121735\n",
      "Episode 2 Step 587 Action [[1.26719714]] Reward -9.76840996236201 Loss 0.026626821607351303\n",
      "Episode 2 Step 588 Action [[1.31215256]] Reward -9.339698330461559 Loss 0.052565112709999084\n",
      "Episode 2 Step 589 Action [[1.17816267]] Reward -9.001323572168358 Loss 0.020906317979097366\n",
      "Episode 2 Step 590 Action [[0.54580816]] Reward -8.766598340275852 Loss 0.018319081515073776\n",
      "Episode 2 Step 591 Action [[0.41177406]] Reward -8.622314690465211 Loss 0.07743188738822937\n",
      "Episode 2 Step 592 Action [[0.60789607]] Reward -8.55247823517828 Loss 0.012166930362582207\n",
      "Episode 2 Step 593 Action [[0.51509743]] Reward -8.566183917610928 Loss 0.014042176306247711\n",
      "Episode 2 Step 594 Action [[0.96810132]] Reward -8.662165525319908 Loss 0.015491381287574768\n",
      "Episode 2 Step 595 Action [[1.18439204]] Reward -8.863261556499273 Loss 0.0426582470536232\n",
      "Episode 2 Step 596 Action [[1.238234]] Reward -9.176725626516049 Loss 0.01629946008324623\n",
      "Episode 2 Step 597 Action [[1.14902994]] Reward -9.592326580526114 Loss 0.008680444210767746\n",
      "Episode 2 Step 598 Action [[1.23832682]] Reward -10.048602210034106 Loss 0.01669418253004551\n",
      "Episode 2 Step 599 Action [[1.23840248]] Reward -9.613191506736488 Loss 0.013509745709598064\n",
      "TOTAL REWARD @ 2-th Episode  : Reward -1597.3226227726298\n",
      "Total Step: 600\n",
      "\n",
      "Episode : 3 Replay Buffer 600\n",
      "[-0.9539492 -0.2999682 -0.2873326]\n",
      "Episode 3 Step 600 Action [[1.04235942]] Reward -8.057533279773038 Loss 0.0507969968020916\n",
      "Episode 3 Step 601 Action [[0.93687731]] Reward -8.163037662263887 Loss 0.014431532472372055\n",
      "Episode 3 Step 602 Action [[0.92139845]] Reward -8.291159596872042 Loss 0.037232473492622375\n",
      "Episode 3 Step 603 Action [[0.86084234]] Reward -8.436831646175875 Loss 0.038369398564100266\n",
      "Episode 3 Step 604 Action [[0.93755747]] Reward -8.598322324192184 Loss 0.01927642524242401\n",
      "Episode 3 Step 605 Action [[0.825848]] Reward -8.76345310571283 Loss 0.026647262275218964\n",
      "Episode 3 Step 606 Action [[0.72269077]] Reward -8.934370316258788 Loss 0.01918967068195343\n",
      "Episode 3 Step 607 Action [[0.92344162]] Reward -9.1097104785574 Loss 0.03482382372021675\n",
      "Episode 3 Step 608 Action [[0.91959274]] Reward -9.267828646629843 Loss 0.011795028112828732\n",
      "Episode 3 Step 609 Action [[0.34916764]] Reward -9.406307477660938 Loss 0.014775436371564865\n",
      "Episode 3 Step 610 Action [[0.45054648]] Reward -9.555521034204833 Loss 0.026889974251389503\n",
      "Episode 3 Step 611 Action [[0.75518924]] Reward -9.694665023687262 Loss 0.011808239854872227\n",
      "Episode 3 Step 612 Action [[0.3523924]] Reward -9.801094412123518 Loss 0.011090768501162529\n",
      "Episode 3 Step 613 Action [[0.20465321]] Reward -9.86077883599855 Loss 0.010417625308036804\n",
      "Episode 3 Step 614 Action [[0.07244133]] Reward -9.767840203464111 Loss 0.008041257970035076\n",
      "Episode 3 Step 615 Action [[0.70705579]] Reward -9.68402184098452 Loss 0.013668365776538849\n",
      "Episode 3 Step 616 Action [[0.581952]] Reward -9.636099070872529 Loss 0.00821570586413145\n",
      "Episode 3 Step 617 Action [[0.50505522]] Reward -9.627559858671228 Loss 0.012644253671169281\n",
      "Episode 3 Step 618 Action [[0.54733171]] Reward -9.654184186451102 Loss 0.008801881223917007\n",
      "Episode 3 Step 619 Action [[0.29170863]] Reward -9.716556933648558 Loss 0.0047583612613379955\n",
      "Episode 3 Step 620 Action [[0.32053071]] Reward -9.798722598738369 Loss 0.0028802333399653435\n",
      "Episode 3 Step 621 Action [[0.52847846]] Reward -9.859340198357302 Loss 0.004644936416298151\n",
      "Episode 3 Step 622 Action [[0.24830738]] Reward -9.743008478617737 Loss 0.0035861353389918804\n",
      "Episode 3 Step 623 Action [[0.01293316]] Reward -9.61754283432303 Loss 0.004873774014413357\n",
      "Episode 3 Step 624 Action [[0.09461971]] Reward -9.498323763076367 Loss 0.005221979692578316\n",
      "Episode 3 Step 625 Action [[0.31756106]] Reward -9.389832081810303 Loss 0.003049938939511776\n",
      "Episode 3 Step 626 Action [[0.5364336]] Reward -9.28715257959606 Loss 0.009626087732613087\n",
      "Episode 3 Step 627 Action [[0.70472208]] Reward -9.183835186984002 Loss 0.003965866286307573\n",
      "Episode 3 Step 628 Action [[0.78840567]] Reward -9.075591924233397 Loss 0.003242489881813526\n",
      "Episode 3 Step 629 Action [[0.9887262]] Reward -8.962255190308888 Loss 0.008926204405725002\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 3 Step 630 Action [[0.76123612]] Reward -8.839632303294307 Loss 0.00995137169957161\n",
      "Episode 3 Step 631 Action [[0.97557549]] Reward -8.718713213315969 Loss 0.006966857239603996\n",
      "Episode 3 Step 632 Action [[0.94890315]] Reward -8.598622808147766 Loss 0.009473523125052452\n",
      "Episode 3 Step 633 Action [[0.91270184]] Reward -8.482379883871305 Loss 0.005564188119024038\n",
      "Episode 3 Step 634 Action [[1.00473545]] Reward -8.376006899059657 Loss 0.01074802316725254\n",
      "Episode 3 Step 635 Action [[0.95761896]] Reward -8.280292599846286 Loss 0.0029960365500301123\n",
      "Episode 3 Step 636 Action [[0.94856702]] Reward -8.199680859821555 Loss 0.0029931191820651293\n",
      "Episode 3 Step 637 Action [[1.23497089]] Reward -8.138601917834842 Loss 0.0045489054173231125\n",
      "Episode 3 Step 638 Action [[1.27671056]] Reward -8.087434114283104 Loss 0.006823827512562275\n",
      "Episode 3 Step 639 Action [[1.38638555]] Reward -8.045030667474915 Loss 0.009843792766332626\n",
      "Episode 3 Step 640 Action [[1.4037248]] Reward -8.00801009473348 Loss 0.007829517126083374\n",
      "Episode 3 Step 641 Action [[1.58321542]] Reward -7.97721117466742 Loss 0.009961435571312904\n",
      "Episode 3 Step 642 Action [[1.54799579]] Reward -7.945499018308552 Loss 0.01109360158443451\n",
      "Episode 3 Step 643 Action [[1.54666282]] Reward -7.915493841754891 Loss 0.02186114899814129\n",
      "Episode 3 Step 644 Action [[1.42509344]] Reward -7.887970243834539 Loss 0.0029509575106203556\n",
      "Episode 3 Step 645 Action [[1.27069684]] Reward -7.8690497822498875 Loss 0.024237042292952538\n",
      "Episode 3 Step 646 Action [[0.93671337]] Reward -7.865972156662248 Loss 0.00822623074054718\n",
      "Episode 3 Step 647 Action [[1.12632966]] Reward -7.895630759240551 Loss 0.006454005837440491\n",
      "Episode 3 Step 648 Action [[1.4003701]] Reward -7.948005702511618 Loss 0.0062553295865654945\n",
      "Episode 3 Step 649 Action [[1.35450937]] Reward -8.006671771817357 Loss 0.008212228305637836\n",
      "Episode 3 Step 650 Action [[0.904869]] Reward -8.072689654230146 Loss 0.008008120581507683\n",
      "Episode 3 Step 651 Action [[0.95086708]] Reward -8.168470677127974 Loss 0.007151234894990921\n",
      "Episode 3 Step 652 Action [[1.013729]] Reward -8.284563043238292 Loss 0.004494595341384411\n",
      "Episode 3 Step 653 Action [[0.8353984]] Reward -8.4128913910665 Loss 0.012865042313933372\n",
      "Episode 3 Step 654 Action [[0.59630023]] Reward -8.560033140477318 Loss 0.038553424179553986\n",
      "Episode 3 Step 655 Action [[0.65729072]] Reward -8.733298611729026 Loss 0.004140935838222504\n",
      "Episode 3 Step 656 Action [[0.65056219]] Reward -8.918678231224078 Loss 0.0030011278577148914\n",
      "Episode 3 Step 657 Action [[1.0055967]] Reward -9.111063244183788 Loss 0.009333048015832901\n",
      "Episode 3 Step 658 Action [[0.79341584]] Reward -9.278808558171516 Loss 0.0032504876144230366\n",
      "Episode 3 Step 659 Action [[0.75852337]] Reward -9.43691317472679 Loss 0.012815302237868309\n",
      "Episode 3 Step 660 Action [[0.64256409]] Reward -9.577336755302882 Loss 0.005946324206888676\n",
      "Episode 3 Step 661 Action [[0.41267786]] Reward -9.701251171136226 Loss 0.009158804081380367\n",
      "Episode 3 Step 662 Action [[0.41066843]] Reward -9.81594445960192 Loss 0.008912636898458004\n",
      "Episode 3 Step 663 Action [[0.34065239]] Reward -9.8466385852975 Loss 0.008708406239748001\n",
      "Episode 3 Step 664 Action [[0.51965478]] Reward -9.759348629148361 Loss 0.006405320949852467\n",
      "Episode 3 Step 665 Action [[0.2641083]] Reward -9.699871646343588 Loss 0.007428515702486038\n",
      "Episode 3 Step 666 Action [[0.68321561]] Reward -9.662229118654354 Loss 0.0077498070895671844\n",
      "Episode 3 Step 667 Action [[0.46848467]] Reward -9.664055086379152 Loss 0.0063846823759377\n",
      "Episode 3 Step 668 Action [[0.0295758]] Reward -9.697916044799912 Loss 0.006453976035118103\n",
      "Episode 3 Step 669 Action [[0.22384854]] Reward -9.739433965332518 Loss 0.006441953592002392\n",
      "Episode 3 Step 670 Action [[0.07648719]] Reward -9.797333777084194 Loss 0.010136290453374386\n",
      "Episode 3 Step 671 Action [[0.413045]] Reward -9.861286264050229 Loss 0.007429758086800575\n",
      "Episode 3 Step 672 Action [[0.37657294]] Reward -9.806317988969408 Loss 0.00869777612388134\n",
      "Episode 3 Step 673 Action [[0.48772369]] Reward -9.71160147855375 Loss 0.010093172080814838\n",
      "Episode 3 Step 674 Action [[0.58286818]] Reward -9.601776366453635 Loss 0.0059843892231583595\n",
      "Episode 3 Step 675 Action [[0.40605979]] Reward -9.476462998401303 Loss 0.008704092353582382\n",
      "Episode 3 Step 676 Action [[0.47815829]] Reward -9.345552564168317 Loss 0.004848083946853876\n",
      "Episode 3 Step 677 Action [[0.22332153]] Reward -9.213039205004739 Loss 0.01769893430173397\n",
      "Episode 3 Step 678 Action [[0.18128796]] Reward -9.091700324540794 Loss 0.005855460651218891\n",
      "Episode 3 Step 679 Action [[0.3649725]] Reward -8.991053320852346 Loss 0.02056550234556198\n",
      "Episode 3 Step 680 Action [[-0.03205421]] Reward -8.908397039882415 Loss 0.021421175450086594\n",
      "Episode 3 Step 681 Action [[0.3598229]] Reward -8.860998885793155 Loss 0.027258088812232018\n",
      "Episode 3 Step 682 Action [[0.37041764]] Reward -8.837710115444683 Loss 0.011529632844030857\n",
      "Episode 3 Step 683 Action [[0.51401021]] Reward -8.836471768558678 Loss 0.013979887589812279\n",
      "Episode 3 Step 684 Action [[0.8586947]] Reward -8.850917516259774 Loss 0.014260152354836464\n",
      "Episode 3 Step 685 Action [[0.77616644]] Reward -8.863217678665764 Loss 0.009392587468028069\n",
      "Episode 3 Step 686 Action [[1.24429082]] Reward -8.878633007723367 Loss 0.008160607889294624\n",
      "Episode 3 Step 687 Action [[1.15698536]] Reward -8.873110712813261 Loss 0.006652611307799816\n",
      "Episode 3 Step 688 Action [[1.07659021]] Reward -8.852851612019666 Loss 0.018000338226556778\n",
      "Episode 3 Step 689 Action [[1.18971413]] Reward -8.822207933114464 Loss 0.005698445253074169\n",
      "Episode 3 Step 690 Action [[0.95979332]] Reward -8.776650506655198 Loss 0.006854014936834574\n",
      "Episode 3 Step 691 Action [[1.14245839]] Reward -8.728149418983804 Loss 0.0038953281473368406\n",
      "Episode 3 Step 692 Action [[1.10484823]] Reward -8.670933474005102 Loss 0.00381480879150331\n",
      "Episode 3 Step 693 Action [[1.15322297]] Reward -8.608350561417526 Loss 0.005088856443762779\n",
      "Episode 3 Step 694 Action [[1.04967583]] Reward -8.540540868690332 Loss 0.00901347678154707\n",
      "Episode 3 Step 695 Action [[1.20229387]] Reward -8.474358476253203 Loss 0.008476851508021355\n",
      "Episode 3 Step 696 Action [[1.53913743]] Reward -8.407323983518074 Loss 0.007996054366230965\n",
      "Episode 3 Step 697 Action [[1.6659222]] Reward -8.327825171761292 Loss 0.0037463237531483173\n",
      "Episode 3 Step 698 Action [[1.70119493]] Reward -8.232684463042524 Loss 0.016184478998184204\n",
      "Episode 3 Step 699 Action [[1.44127334]] Reward -8.12262410724026 Loss 0.007414562627673149\n",
      "Episode 3 Step 700 Action [[1.45566863]] Reward -8.010721475452426 Loss 0.01070091500878334\n",
      "Episode 3 Step 701 Action [[1.21976156]] Reward -7.901812595066522 Loss 0.009979971684515476\n",
      "Episode 3 Step 702 Action [[1.07433529]] Reward -7.808051159329142 Loss 0.01928762160241604\n",
      "Episode 3 Step 703 Action [[1.04481252]] Reward -7.740817004861828 Loss 0.014857940375804901\n",
      "Episode 3 Step 704 Action [[1.10339893]] Reward -7.705437539515303 Loss 0.014025012031197548\n",
      "Episode 3 Step 705 Action [[1.08444891]] Reward -7.70085197882094 Loss 0.005764488130807877\n",
      "Episode 3 Step 706 Action [[0.92548913]] Reward -7.727685877221356 Loss 0.009761911816895008\n",
      "Episode 3 Step 707 Action [[1.14389699]] Reward -7.793810093409414 Loss 0.04332926869392395\n",
      "Episode 3 Step 708 Action [[0.99842969]] Reward -7.884161431027592 Loss 0.02567826211452484\n",
      "Episode 3 Step 709 Action [[0.71542012]] Reward -8.004630806063556 Loss 0.029406337067484856\n",
      "Episode 3 Step 710 Action [[0.56363191]] Reward -8.166701360076754 Loss 0.057124070823192596\n",
      "Episode 3 Step 711 Action [[0.43724134]] Reward -8.371088637712898 Loss 0.00983419269323349\n",
      "Episode 3 Step 712 Action [[0.60444197]] Reward -8.616815122333241 Loss 0.006431092973798513\n",
      "Episode 3 Step 713 Action [[0.68526956]] Reward -8.880109874249984 Loss 0.03356111794710159\n",
      "Episode 3 Step 714 Action [[0.82831862]] Reward -9.14844002737456 Loss 0.06412647664546967\n",
      "Episode 3 Step 715 Action [[0.95936717]] Reward -9.40300822620054 Loss 0.03602924197912216\n",
      "Episode 3 Step 716 Action [[1.03816186]] Reward -9.628724156430344 Loss 0.02567373961210251\n",
      "Episode 3 Step 717 Action [[1.33065251]] Reward -9.816333684354598 Loss 0.06423743069171906\n",
      "Episode 3 Step 718 Action [[0.79606243]] Reward -9.843456774613827 Loss 0.017258919775485992\n",
      "Episode 3 Step 719 Action [[0.52730795]] Reward -9.725751317295886 Loss 0.012693365104496479\n",
      "Episode 3 Step 720 Action [[0.44705964]] Reward -9.643649185313937 Loss 0.011014197021722794\n",
      "Episode 3 Step 721 Action [[0.14449001]] Reward -9.593120251291673 Loss 0.027643155306577682\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 3 Step 722 Action [[0.24795423]] Reward -9.562377807637718 Loss 0.013645251281559467\n",
      "Episode 3 Step 723 Action [[0.32424796]] Reward -9.555116600030525 Loss 0.010110029950737953\n",
      "Episode 3 Step 724 Action [[0.49824154]] Reward -9.57594770596592 Loss 0.013263814151287079\n",
      "Episode 3 Step 725 Action [[0.48829633]] Reward -9.633061174391399 Loss 0.01626651920378208\n",
      "Episode 3 Step 726 Action [[0.5657349]] Reward -9.72417308294377 Loss 0.011375678703188896\n",
      "Episode 3 Step 727 Action [[0.30463293]] Reward -9.849889810478432 Loss 0.008212367072701454\n",
      "Episode 3 Step 728 Action [[0.40002421]] Reward -9.787587474520713 Loss 0.010330268181860447\n",
      "Episode 3 Step 729 Action [[0.86579381]] Reward -9.643461531464501 Loss 0.02025497518479824\n",
      "Episode 3 Step 730 Action [[1.17427376]] Reward -9.476010221766394 Loss 0.012335183098912239\n",
      "Episode 3 Step 731 Action [[1.06724837]] Reward -9.276911544168374 Loss 0.010753227397799492\n",
      "Episode 3 Step 732 Action [[0.98040935]] Reward -9.051972403472757 Loss 0.009766684845089912\n",
      "Episode 3 Step 733 Action [[0.94290987]] Reward -8.812618198149313 Loss 0.020660102367401123\n",
      "Episode 3 Step 734 Action [[1.41458603]] Reward -8.57095350339061 Loss 0.013891949318349361\n",
      "Episode 3 Step 735 Action [[1.34182647]] Reward -8.325630802715361 Loss 0.007155377417802811\n",
      "Episode 3 Step 736 Action [[1.05046086]] Reward -8.075967790938329 Loss 0.010010217316448689\n",
      "Episode 3 Step 737 Action [[0.691738]] Reward -7.83813850305362 Loss 0.008223684504628181\n",
      "Episode 3 Step 738 Action [[0.77523082]] Reward -7.636967930477039 Loss 0.01124428678303957\n",
      "Episode 3 Step 739 Action [[1.00072114]] Reward -7.487294621406496 Loss 0.006416105665266514\n",
      "Episode 3 Step 740 Action [[0.95978168]] Reward -7.385205860029124 Loss 0.005448075011372566\n",
      "Episode 3 Step 741 Action [[1.03396612]] Reward -7.332599783743046 Loss 0.034864090383052826\n",
      "Episode 3 Step 742 Action [[1.10541868]] Reward -7.3290515932542775 Loss 0.017919214442372322\n",
      "Episode 3 Step 743 Action [[1.13701744]] Reward -7.3709068702392235 Loss 0.007266858592629433\n",
      "Episode 3 Step 744 Action [[1.1796796]] Reward -7.455047915340054 Loss 0.00875340960919857\n",
      "Episode 3 Step 745 Action [[0.97389309]] Reward -7.5756957438055625 Loss 0.02302960865199566\n",
      "Episode 3 Step 746 Action [[1.07965201]] Reward -7.741768476456644 Loss 0.004108930006623268\n",
      "Episode 3 Step 747 Action [[1.27896386]] Reward -7.938505152559309 Loss 0.002289353869855404\n",
      "Episode 3 Step 748 Action [[0.88946563]] Reward -8.145471570721588 Loss 0.003565337974578142\n",
      "Episode 3 Step 749 Action [[0.81004507]] Reward -8.38609323516954 Loss 0.0047010378912091255\n",
      "Episode 3 Step 750 Action [[0.66624744]] Reward -8.648523562958198 Loss 0.004356492310762405\n",
      "Episode 3 Step 751 Action [[0.21263015]] Reward -8.931151268115412 Loss 0.004435992334038019\n",
      "Episode 3 Step 752 Action [[0.58191474]] Reward -9.254366021521918 Loss 0.009842613711953163\n",
      "Episode 3 Step 753 Action [[0.8618778]] Reward -9.56477773460029 Loss 0.006298739463090897\n",
      "Episode 3 Step 754 Action [[1.12124193]] Reward -9.839858135001476 Loss 0.004295193590223789\n",
      "Episode 3 Step 755 Action [[0.71059102]] Reward -9.80497402057041 Loss 0.02280634641647339\n",
      "Episode 3 Step 756 Action [[1.08760971]] Reward -9.582972830637559 Loss 0.01045028306543827\n",
      "Episode 3 Step 757 Action [[1.64421707]] Reward -9.420991516112645 Loss 0.006647118367254734\n",
      "Episode 3 Step 758 Action [[1.42385577]] Reward -9.354610889365036 Loss 0.005261626560240984\n",
      "Episode 3 Step 759 Action [[1.32803513]] Reward -9.390935770993556 Loss 0.010935050435364246\n",
      "Episode 3 Step 760 Action [[1.29973765]] Reward -9.520509131716867 Loss 0.03275521472096443\n",
      "Episode 3 Step 761 Action [[1.02538682]] Reward -9.735432507724198 Loss 0.011475970968604088\n",
      "Episode 3 Step 762 Action [[0.77686836]] Reward -9.858329971917772 Loss 0.005170751363039017\n",
      "Episode 3 Step 763 Action [[0.78640416]] Reward -9.596646658653237 Loss 0.010613659396767616\n",
      "Episode 3 Step 764 Action [[0.60607854]] Reward -9.311794484024789 Loss 0.0054643722251057625\n",
      "Episode 3 Step 765 Action [[0.96706389]] Reward -9.017201581989953 Loss 0.006713473238050938\n",
      "Episode 3 Step 766 Action [[1.2373034]] Reward -8.722499490736553 Loss 0.007281297352164984\n",
      "Episode 3 Step 767 Action [[1.28644048]] Reward -8.424024461694069 Loss 0.0027905418537557125\n",
      "Episode 3 Step 768 Action [[1.34488375]] Reward -8.124345831831477 Loss 0.00795629434287548\n",
      "Episode 3 Step 769 Action [[1.31794044]] Reward -7.831656615915137 Loss 0.001782405306585133\n",
      "Episode 3 Step 770 Action [[1.31064823]] Reward -7.555128764678789 Loss 0.0069193230010569096\n",
      "Episode 3 Step 771 Action [[1.33421662]] Reward -7.3056838394861785 Loss 0.002046996494755149\n",
      "Episode 3 Step 772 Action [[1.07242744]] Reward -7.091366876015568 Loss 0.0034384960308670998\n",
      "Episode 3 Step 773 Action [[1.16422176]] Reward -6.926738335149097 Loss 0.004413493908941746\n",
      "Episode 3 Step 774 Action [[1.30094375]] Reward -6.821515790738673 Loss 0.0027555807027965784\n",
      "Episode 3 Step 775 Action [[1.22431672]] Reward -6.77239809287924 Loss 0.01206151582300663\n",
      "Episode 3 Step 776 Action [[1.46232522]] Reward -6.783726500496408 Loss 0.007179529406130314\n",
      "Episode 3 Step 777 Action [[1.34578726]] Reward -6.843503367339249 Loss 0.004132514353841543\n",
      "Episode 3 Step 778 Action [[1.40748305]] Reward -6.956316361158235 Loss 0.005324272904545069\n",
      "Episode 3 Step 779 Action [[1.25019978]] Reward -7.1137449352228295 Loss 0.0038364259526133537\n",
      "Episode 3 Step 780 Action [[1.36843317]] Reward -7.3216652179531785 Loss 0.010549805127084255\n",
      "Episode 3 Step 781 Action [[1.56440765]] Reward -7.562619327472311 Loss 0.0021043363958597183\n",
      "Episode 3 Step 782 Action [[1.50953228]] Reward -7.815305295711642 Loss 0.034938909113407135\n",
      "Episode 3 Step 783 Action [[1.39088388]] Reward -8.078988735199172 Loss 0.00858365185558796\n",
      "Episode 3 Step 784 Action [[1.06107502]] Reward -8.350623228027532 Loss 0.012320224195718765\n",
      "Episode 3 Step 785 Action [[0.71317725]] Reward -8.641425850799855 Loss 0.01112367957830429\n",
      "Episode 3 Step 786 Action [[0.88393765]] Reward -8.958128938703625 Loss 0.02505354955792427\n",
      "Episode 3 Step 787 Action [[1.15323557]] Reward -9.265118332078032 Loss 0.0044638668186962605\n",
      "Episode 3 Step 788 Action [[0.98350178]] Reward -9.534440973194558 Loss 0.0191195011138916\n",
      "Episode 3 Step 789 Action [[0.8886389]] Reward -9.77819846807561 Loss 0.004127795808017254\n",
      "Episode 3 Step 790 Action [[0.68940181]] Reward -9.858204042169604 Loss 0.011297960765659809\n",
      "Episode 3 Step 791 Action [[0.65550837]] Reward -9.650310807607868 Loss 0.007766024209558964\n",
      "Episode 3 Step 792 Action [[1.02618176]] Reward -9.485402870151036 Loss 0.02395506203174591\n",
      "Episode 3 Step 793 Action [[1.10188561]] Reward -9.381917873253064 Loss 0.023564808070659637\n",
      "Episode 3 Step 794 Action [[1.23901314]] Reward -9.355733046279896 Loss 0.04266553372144699\n",
      "Episode 3 Step 795 Action [[1.00719146]] Reward -9.415661577056063 Loss 0.022437062114477158\n",
      "Episode 3 Step 796 Action [[1.40358516]] Reward -9.549716784599319 Loss 0.0440949983894825\n",
      "Episode 3 Step 797 Action [[1.04894271]] Reward -9.774997153603042 Loss 0.023511525243520737\n",
      "Episode 3 Step 798 Action [[0.9920395]] Reward -9.818559598015481 Loss 0.02378416433930397\n",
      "Episode 3 Step 799 Action [[1.01514197]] Reward -9.547126321815508 Loss 0.04387303441762924\n",
      "Now we save model\n",
      "TOTAL REWARD @ 3-th Episode  : Reward -1756.9493826383884\n",
      "Total Step: 800\n",
      "\n",
      "Episode : 4 Replay Buffer 800\n",
      "[ 0.93832731 -0.34574826  0.55996975]\n",
      "Episode 4 Step 800 Action [[0.10794399]] Reward -0.15600275844115782 Loss 0.012594674713909626\n",
      "Episode 4 Step 801 Action [[0.00732885]] Reward -0.12373898091572894 Loss 0.06092803180217743\n",
      "Episode 4 Step 802 Action [[0.01127849]] Reward -0.11184504806777991 Loss 0.011202031746506691\n",
      "Episode 4 Step 803 Action [[-0.11584]] Reward -0.12029069742041526 Loss 0.06262322515249252\n",
      "Episode 4 Step 804 Action [[-0.0080422]] Reward -0.15257159453269942 Loss 0.0038368841633200645\n",
      "Episode 4 Step 805 Action [[0.20342862]] Reward -0.2108706799265615 Loss 0.05138704925775528\n",
      "Episode 4 Step 806 Action [[0.15339068]] Reward -0.2962922072314937 Loss 0.03158669173717499\n",
      "Episode 4 Step 807 Action [[0.50009946]] Reward -0.4259082729915258 Loss 0.03069080039858818\n",
      "Episode 4 Step 808 Action [[0.46527241]] Reward -0.5957802097141106 Loss 0.014393369667232037\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 4 Step 809 Action [[0.55162066]] Reward -0.8397511286679592 Loss 0.021459020674228668\n",
      "Episode 4 Step 810 Action [[0.55218895]] Reward -1.1778643008811855 Loss 0.05838839337229729\n",
      "Episode 4 Step 811 Action [[0.70838367]] Reward -1.650922851403044 Loss 0.02743176743388176\n",
      "Episode 4 Step 812 Action [[0.81760624]] Reward -2.2865342571216796 Loss 0.14965635538101196\n",
      "Episode 4 Step 813 Action [[0.61101043]] Reward -3.130994573698177 Loss 0.14366185665130615\n",
      "Episode 4 Step 814 Action [[0.98608313]] Reward -4.276402614891445 Loss 0.09689624607563019\n",
      "Episode 4 Step 815 Action [[1.28628984]] Reward -5.676104633582311 Loss 0.0789775550365448\n",
      "Episode 4 Step 816 Action [[1.26242069]] Reward -7.319019985355042 Loss 0.1773242950439453\n",
      "Episode 4 Step 817 Action [[1.60316899]] Reward -9.209321096440169 Loss 0.042391594499349594\n",
      "Episode 4 Step 818 Action [[1.59071851]] Reward -11.167846892207766 Loss 0.2062595635652542\n",
      "Episode 4 Step 819 Action [[1.43806059]] Reward -13.12694837810277 Loss 0.045568302273750305\n",
      "Episode 4 Step 820 Action [[1.39331342]] Reward -12.878775661481574 Loss 0.4578242897987366\n",
      "Episode 4 Step 821 Action [[1.51339618]] Reward -10.765930112326604 Loss 0.0330987274646759\n",
      "Episode 4 Step 822 Action [[1.40694006]] Reward -8.73319071111353 Loss 0.33328646421432495\n",
      "Episode 4 Step 823 Action [[1.43653437]] Reward -6.921750554759234 Loss 0.06879264861345291\n",
      "Episode 4 Step 824 Action [[0.95008359]] Reward -5.3924311333322015 Loss 0.10854292660951614\n",
      "Episode 4 Step 825 Action [[1.15376389]] Reward -4.232952009802993 Loss 0.22228066623210907\n",
      "Episode 4 Step 826 Action [[1.22161764]] Reward -3.3816996900504788 Loss 0.04934779182076454\n",
      "Episode 4 Step 827 Action [[1.01051498]] Reward -2.859359554614845 Loss 0.19937652349472046\n",
      "Episode 4 Step 828 Action [[0.7980078]] Reward -2.66251627941834 Loss 0.0915929451584816\n",
      "Episode 4 Step 829 Action [[0.80238886]] Reward -2.766873765784184 Loss 0.04219987988471985\n",
      "Episode 4 Step 830 Action [[0.7321871]] Reward -3.1709411263724476 Loss 0.12371189147233963\n",
      "Episode 4 Step 831 Action [[0.67137865]] Reward -3.8823765797187275 Loss 0.015382355079054832\n",
      "Episode 4 Step 832 Action [[0.54822016]] Reward -4.909810145460527 Loss 0.0738254189491272\n",
      "Episode 4 Step 833 Action [[0.96436483]] Reward -6.2428336387070456 Loss 0.09596558660268784\n",
      "Episode 4 Step 834 Action [[0.88904954]] Reward -7.94022343851385 Loss 0.04074431210756302\n",
      "Episode 4 Step 835 Action [[1.07572238]] Reward -9.894019637671999 Loss 0.10353943705558777\n",
      "Episode 4 Step 836 Action [[1.31507409]] Reward -12.056996970876007 Loss 0.02652868628501892\n",
      "Episode 4 Step 837 Action [[1.40642977]] Reward -12.791195228022252 Loss 0.05012704059481621\n",
      "Episode 4 Step 838 Action [[1.74063909]] Reward -11.167165038342988 Loss 0.04534690082073212\n",
      "Episode 4 Step 839 Action [[1.9255748]] Reward -9.535670207638212 Loss 0.01874331757426262\n",
      "Episode 4 Step 840 Action [[2.1513425]] Reward -7.942063589101762 Loss 0.04471202939748764\n",
      "Episode 4 Step 841 Action [[1.7744078]] Reward -6.449437515792436 Loss 0.009973660111427307\n",
      "Episode 4 Step 842 Action [[1.68118938]] Reward -5.089644921957997 Loss 0.015335414558649063\n",
      "Episode 4 Step 843 Action [[1.57180495]] Reward -3.93278742786724 Loss 0.024521350860595703\n",
      "Episode 4 Step 844 Action [[1.338845]] Reward -2.9896561966464033 Loss 0.007880086079239845\n",
      "Episode 4 Step 845 Action [[0.72550345]] Reward -2.235901425379755 Loss 0.023177357390522957\n",
      "Episode 4 Step 846 Action [[0.52003187]] Reward -1.6199566215218895 Loss 0.006083015352487564\n",
      "Episode 4 Step 847 Action [[0.57709351]] Reward -1.157623286080239 Loss 0.03533168137073517\n",
      "Episode 4 Step 848 Action [[0.64228363]] Reward -0.8311104585342444 Loss 0.02247730828821659\n",
      "Episode 4 Step 849 Action [[0.38292689]] Reward -0.6036475846281917 Loss 0.0049628568813204765\n",
      "Episode 4 Step 850 Action [[0.34820093]] Reward -0.4328161347071349 Loss 0.011273323558270931\n",
      "Episode 4 Step 851 Action [[0.62118766]] Reward -0.31405943166812156 Loss 0.01043433602899313\n",
      "Episode 4 Step 852 Action [[0.59636803]] Reward -0.24402550063441827 Loss 0.0067615513689816\n",
      "Episode 4 Step 853 Action [[0.28503381]] Reward -0.1998251473213497 Loss 0.036816954612731934\n",
      "Episode 4 Step 854 Action [[0.43724416]] Reward -0.16708860391625568 Loss 0.017914853990077972\n",
      "Episode 4 Step 855 Action [[0.61170188]] Reward -0.16050590186507252 Loss 0.007777306716889143\n",
      "Episode 4 Step 856 Action [[0.45314824]] Reward -0.18096968929591614 Loss 0.01725330948829651\n",
      "Episode 4 Step 857 Action [[0.53980679]] Reward -0.2203788835656546 Loss 0.012804783880710602\n",
      "Episode 4 Step 858 Action [[0.65890984]] Reward -0.2963907782706921 Loss 0.014413353055715561\n",
      "Episode 4 Step 859 Action [[0.26453179]] Reward -0.42528465547739436 Loss 0.025012096390128136\n",
      "Episode 4 Step 860 Action [[0.47082963]] Reward -0.5969819474980932 Loss 0.010152235627174377\n",
      "Episode 4 Step 861 Action [[0.75790796]] Reward -0.8726087150625498 Loss 0.011024368926882744\n",
      "Episode 4 Step 862 Action [[0.82042948]] Reward -1.3079771780237508 Loss 0.01326388493180275\n",
      "Episode 4 Step 863 Action [[0.42101542]] Reward -1.951537436388088 Loss 0.0036651319824159145\n",
      "Episode 4 Step 864 Action [[0.59799595]] Reward -2.820948985103976 Loss 0.010098427534103394\n",
      "Episode 4 Step 865 Action [[0.95289102]] Reward -4.062098076140841 Loss 0.0049010771326720715\n",
      "Episode 4 Step 866 Action [[0.54734132]] Reward -5.8075257823287405 Loss 0.010366726666688919\n",
      "Episode 4 Step 867 Action [[0.33303363]] Reward -7.983216337628731 Loss 0.02111401967704296\n",
      "Episode 4 Step 868 Action [[0.57950423]] Reward -10.575510602318024 Loss 0.00752330431714654\n",
      "Episode 4 Step 869 Action [[0.72114438]] Reward -13.563769926970565 Loss 0.007013267371803522\n",
      "Episode 4 Step 870 Action [[0.37157887]] Reward -16.001800223865096 Loss 0.07327458262443542\n",
      "Episode 4 Step 871 Action [[0.65364439]] Reward -14.129571754142631 Loss 0.03296913206577301\n",
      "Episode 4 Step 872 Action [[0.565278]] Reward -11.840521247396737 Loss 0.018891505897045135\n",
      "Episode 4 Step 873 Action [[0.9247069]] Reward -9.557155057775768 Loss 0.038622528314590454\n",
      "Episode 4 Step 874 Action [[1.05558348]] Reward -7.52227643062496 Loss 0.06104625016450882\n",
      "Episode 4 Step 875 Action [[1.220254]] Reward -5.79695069738487 Loss 0.029198039323091507\n",
      "Episode 4 Step 876 Action [[0.81608626]] Reward -4.437413282835485 Loss 0.050779081881046295\n",
      "Episode 4 Step 877 Action [[0.85420105]] Reward -3.3604455158526507 Loss 0.030083345249295235\n",
      "Episode 4 Step 878 Action [[0.61194058]] Reward -2.611837587623418 Loss 0.11542275547981262\n",
      "Episode 4 Step 879 Action [[0.2885249]] Reward -2.1119291845868786 Loss 0.07755502313375473\n",
      "Episode 4 Step 880 Action [[0.43169913]] Reward -1.8191133771325527 Loss 0.18766266107559204\n",
      "Episode 4 Step 881 Action [[0.55354248]] Reward -1.7696147234488115 Loss 0.13175958395004272\n",
      "Episode 4 Step 882 Action [[0.72431893]] Reward -1.961589205188536 Loss 0.18018746376037598\n",
      "Episode 4 Step 883 Action [[0.60612843]] Reward -2.4362356623931825 Loss 0.16730815172195435\n",
      "Episode 4 Step 884 Action [[0.6271476]] Reward -3.2190655358098175 Loss 0.15854430198669434\n",
      "Episode 4 Step 885 Action [[0.60180646]] Reward -4.416725598741522 Loss 0.10176479071378708\n",
      "Episode 4 Step 886 Action [[0.48232826]] Reward -6.119438974142189 Loss 0.09007901698350906\n",
      "Episode 4 Step 887 Action [[0.65222688]] Reward -8.377152838424823 Loss 0.08388306945562363\n",
      "Episode 4 Step 888 Action [[0.73590513]] Reward -11.048883760341257 Loss 0.11671608686447144\n",
      "Episode 4 Step 889 Action [[0.72296513]] Reward -13.123034346011936 Loss 0.0298411063849926\n",
      "Episode 4 Step 890 Action [[0.83481266]] Reward -15.507606716148212 Loss 0.03690669313073158\n",
      "Episode 4 Step 891 Action [[0.82827248]] Reward -14.549009638608137 Loss 0.05411865934729576\n",
      "Episode 4 Step 892 Action [[1.09593155]] Reward -12.307344935253479 Loss 0.03147519752383232\n",
      "Episode 4 Step 893 Action [[0.95669486]] Reward -10.102985911549622 Loss 0.0644974410533905\n",
      "Episode 4 Step 894 Action [[1.07613328]] Reward -8.002763846681262 Loss 0.06868923455476761\n",
      "Episode 4 Step 895 Action [[1.02298558]] Reward -6.19726542831142 Loss 0.06980402022600174\n",
      "Episode 4 Step 896 Action [[0.91394828]] Reward -4.727826892259774 Loss 0.018815932795405388\n",
      "Episode 4 Step 897 Action [[0.87689679]] Reward -3.603558541283354 Loss 0.013295733369886875\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 4 Step 898 Action [[0.78641818]] Reward -2.81262195063028 Loss 0.027243761345744133\n",
      "Episode 4 Step 899 Action [[0.61487917]] Reward -2.308806376781689 Loss 0.06876053661108017\n",
      "Episode 4 Step 900 Action [[0.66302078]] Reward -2.050005301670748 Loss 0.0754706859588623\n",
      "Episode 4 Step 901 Action [[0.84127163]] Reward -2.050366360858018 Loss 0.07189062237739563\n",
      "Episode 4 Step 902 Action [[0.79140398]] Reward -2.3398961939196257 Loss 0.07281865924596786\n",
      "Episode 4 Step 903 Action [[0.98406374]] Reward -2.935499411457348 Loss 0.01594872958958149\n",
      "Episode 4 Step 904 Action [[1.12912084]] Reward -3.9566657504352007 Loss 0.018536772578954697\n",
      "Episode 4 Step 905 Action [[1.09836064]] Reward -5.513927770265457 Loss 0.05765805393457413\n",
      "Episode 4 Step 906 Action [[0.6020579]] Reward -7.6872220946762715 Loss 0.01948307268321514\n",
      "Episode 4 Step 907 Action [[0.3302835]] Reward -10.245395936523902 Loss 0.007252604700624943\n",
      "Episode 4 Step 908 Action [[0.45266722]] Reward -12.151085928917578 Loss 0.040819741785526276\n",
      "Episode 4 Step 909 Action [[0.28783802]] Reward -14.391315531193923 Loss 0.014891532249748707\n",
      "Episode 4 Step 910 Action [[0.43559892]] Reward -15.657249714735421 Loss 0.10293440520763397\n",
      "Episode 4 Step 911 Action [[0.75337696]] Reward -13.37181159262583 Loss 0.07965733110904694\n",
      "Episode 4 Step 912 Action [[0.67222157]] Reward -11.087981975759465 Loss 0.04523922875523567\n",
      "Episode 4 Step 913 Action [[0.32128479]] Reward -8.860153148214987 Loss 0.0316263772547245\n",
      "Episode 4 Step 914 Action [[0.40439943]] Reward -6.799864826865155 Loss 0.035626303404569626\n",
      "Episode 4 Step 915 Action [[0.56341794]] Reward -5.10097029944768 Loss 0.021099822595715523\n",
      "Episode 4 Step 916 Action [[0.32851337]] Reward -3.795011026347649 Loss 0.03582262992858887\n",
      "Episode 4 Step 917 Action [[0.4238156]] Reward -2.7974256617529343 Loss 0.02985355630517006\n",
      "Episode 4 Step 918 Action [[0.57977082]] Reward -2.108136448508369 Loss 0.018898189067840576\n",
      "Episode 4 Step 919 Action [[0.72273092]] Reward -1.6762156977576994 Loss 0.025416571646928787\n",
      "Episode 4 Step 920 Action [[0.75724829]] Reward -1.45680439354348 Loss 0.020298924297094345\n",
      "Episode 4 Step 921 Action [[0.69959123]] Reward -1.418107100266064 Loss 0.024542203173041344\n",
      "Episode 4 Step 922 Action [[0.81294137]] Reward -1.5537598308164948 Loss 0.025962291285395622\n",
      "Episode 4 Step 923 Action [[1.12703845]] Reward -1.9114565767859935 Loss 0.03299325704574585\n",
      "Episode 4 Step 924 Action [[1.25860768]] Reward -2.5789483494172565 Loss 0.11498996615409851\n",
      "Episode 4 Step 925 Action [[1.04279531]] Reward -3.631996558768722 Loss 0.01669042930006981\n",
      "Episode 4 Step 926 Action [[1.36387853]] Reward -5.127480699667605 Loss 0.16105037927627563\n",
      "Episode 4 Step 927 Action [[0.96768838]] Reward -7.271860323710962 Loss 0.027662839740514755\n",
      "Episode 4 Step 928 Action [[0.75916201]] Reward -9.975171989642003 Loss 0.09252835065126419\n",
      "Episode 4 Step 929 Action [[0.63238229]] Reward -12.111013659065138 Loss 0.07137802988290787\n",
      "Episode 4 Step 930 Action [[0.83972102]] Reward -14.3533620648012 Loss 0.23703652620315552\n",
      "Episode 4 Step 931 Action [[0.77529751]] Reward -15.67256848376173 Loss 0.04584486782550812\n",
      "Episode 4 Step 932 Action [[1.15862221]] Reward -13.385695965532745 Loss 0.14159409701824188\n",
      "Episode 4 Step 933 Action [[0.9557604]] Reward -11.180051729367197 Loss 0.16464629769325256\n",
      "Episode 4 Step 934 Action [[0.86652754]] Reward -8.987614666477624 Loss 0.10836531221866608\n",
      "Episode 4 Step 935 Action [[0.89894342]] Reward -6.99689435208551 Loss 0.12486852705478668\n",
      "Episode 4 Step 936 Action [[0.67616568]] Reward -5.340937841855114 Loss 0.1060701385140419\n",
      "Episode 4 Step 937 Action [[1.12549801]] Reward -4.018819103570414 Loss 0.3103332817554474\n",
      "Episode 4 Step 938 Action [[1.1144781]] Reward -3.1193548107435505 Loss 0.03985699266195297\n",
      "Episode 4 Step 939 Action [[1.06197412]] Reward -2.5264148267066022 Loss 0.09787452220916748\n",
      "Episode 4 Step 940 Action [[1.09461902]] Reward -2.20351110960047 Loss 0.13760285079479218\n",
      "Episode 4 Step 941 Action [[1.12674092]] Reward -2.1472099709712547 Loss 0.11369829624891281\n",
      "Episode 4 Step 942 Action [[1.29413315]] Reward -2.369680506533594 Loss 0.11359671503305435\n",
      "Episode 4 Step 943 Action [[1.28804499]] Reward -2.9382633794567594 Loss 0.109949991106987\n",
      "Episode 4 Step 944 Action [[1.4190901]] Reward -3.9152351814583546 Loss 0.20480987429618835\n",
      "Episode 4 Step 945 Action [[1.40236791]] Reward -5.443079456484169 Loss 0.08662493526935577\n",
      "Episode 4 Step 946 Action [[1.13515857]] Reward -7.621595432292211 Loss 0.1092420145869255\n",
      "Episode 4 Step 947 Action [[1.08664318]] Reward -9.997450379531632 Loss 0.19998469948768616\n",
      "Episode 4 Step 948 Action [[1.01161058]] Reward -11.876918118120923 Loss 0.09143060445785522\n",
      "Episode 4 Step 949 Action [[1.11634994]] Reward -14.09960061509674 Loss 0.045507363975048065\n",
      "Episode 4 Step 950 Action [[0.88202933]] Reward -15.928903176998647 Loss 0.07417778670787811\n",
      "Episode 4 Step 951 Action [[1.10327309]] Reward -13.595445616375649 Loss 0.034500524401664734\n",
      "Episode 4 Step 952 Action [[1.0843842]] Reward -11.383333551308978 Loss 0.048975870013237\n",
      "Episode 4 Step 953 Action [[0.77517353]] Reward -9.205984471584635 Loss 0.06017620116472244\n",
      "Episode 4 Step 954 Action [[0.44184133]] Reward -7.169133092959044 Loss 0.02368168905377388\n",
      "Episode 4 Step 955 Action [[0.76713801]] Reward -5.404196529773424 Loss 0.05900655314326286\n",
      "Episode 4 Step 956 Action [[0.61768547]] Reward -4.07173990263611 Loss 0.03858274221420288\n",
      "Episode 4 Step 957 Action [[0.47928992]] Reward -3.0710065156562303 Loss 0.043074365705251694\n",
      "Episode 4 Step 958 Action [[0.47902855]] Reward -2.367197633046225 Loss 0.059470292180776596\n",
      "Episode 4 Step 959 Action [[0.66584423]] Reward -1.9327030678154522 Loss 0.019820397719740868\n",
      "Episode 4 Step 960 Action [[0.77229982]] Reward -1.752579872904211 Loss 0.07442858815193176\n",
      "Episode 4 Step 961 Action [[0.88649451]] Reward -1.803567308331949 Loss 0.03476053476333618\n",
      "Episode 4 Step 962 Action [[0.96009998]] Reward -2.105223386361652 Loss 0.026130225509405136\n",
      "Episode 4 Step 963 Action [[1.00026388]] Reward -2.704416121343655 Loss 0.028749657794833183\n",
      "Episode 4 Step 964 Action [[1.26469862]] Reward -3.6811284197259337 Loss 0.10306400805711746\n",
      "Episode 4 Step 965 Action [[1.17760826]] Reward -5.188343445715209 Loss 0.0471605584025383\n",
      "Episode 4 Step 966 Action [[1.55598414]] Reward -7.284811705590307 Loss 0.013696352951228619\n",
      "Episode 4 Step 967 Action [[1.47139364]] Reward -10.102959891266746 Loss 0.02007284387946129\n",
      "Episode 4 Step 968 Action [[1.4413866]] Reward -12.018830074016847 Loss 0.0461413636803627\n",
      "Episode 4 Step 969 Action [[1.65650729]] Reward -14.281588973153877 Loss 0.014861216768622398\n",
      "Episode 4 Step 970 Action [[1.58121827]] Reward -15.714253251730693 Loss 0.030769184231758118\n",
      "Episode 4 Step 971 Action [[1.34404446]] Reward -13.387417529076135 Loss 0.020306367427110672\n",
      "Episode 4 Step 972 Action [[1.18661972]] Reward -11.219115707273172 Loss 0.02107841894030571\n",
      "Episode 4 Step 973 Action [[1.38320976]] Reward -9.064658284523002 Loss 0.03781694546341896\n",
      "Episode 4 Step 974 Action [[1.27831682]] Reward -7.153458261598007 Loss 0.04926478862762451\n",
      "Episode 4 Step 975 Action [[1.19122906]] Reward -5.536207506400097 Loss 0.04537691920995712\n",
      "Episode 4 Step 976 Action [[1.27835162]] Reward -4.2715353389528605 Loss 0.04597000032663345\n",
      "Episode 4 Step 977 Action [[1.34475381]] Reward -3.3862764359305135 Loss 0.0849662572145462\n",
      "Episode 4 Step 978 Action [[1.36194462]] Reward -2.8489873605568588 Loss 0.03125718608498573\n",
      "Episode 4 Step 979 Action [[1.19854037]] Reward -2.6317233688115222 Loss 0.03808234632015228\n",
      "Episode 4 Step 980 Action [[1.3710908]] Reward -2.7118772823329134 Loss 0.022399377077817917\n",
      "Episode 4 Step 981 Action [[1.43588336]] Reward -3.176212215124902 Loss 0.02349991351366043\n",
      "Episode 4 Step 982 Action [[1.65229354]] Reward -4.089623496942159 Loss 0.02834869921207428\n",
      "Episode 4 Step 983 Action [[1.63217711]] Reward -5.60382233329892 Loss 0.028460301458835602\n",
      "Episode 4 Step 984 Action [[1.43118047]] Reward -7.8150444493191795 Loss 0.010159336030483246\n",
      "Episode 4 Step 985 Action [[1.34853044]] Reward -9.809343997031403 Loss 0.02084704302251339\n",
      "Episode 4 Step 986 Action [[1.74809229]] Reward -11.657015542429207 Loss 0.013642379082739353\n",
      "Episode 4 Step 987 Action [[1.50457678]] Reward -13.873857562616822 Loss 0.046841710805892944\n",
      "Episode 4 Step 988 Action [[1.34355668]] Reward -16.158647295337914 Loss 0.012102195993065834\n",
      "Episode 4 Step 989 Action [[1.28413403]] Reward -13.768490741423957 Loss 0.03274483233690262\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 4 Step 990 Action [[1.2278356]] Reward -11.596581074329878 Loss 0.02531243860721588\n",
      "Episode 4 Step 991 Action [[1.20256664]] Reward -9.431838295971623 Loss 0.03904076665639877\n",
      "Episode 4 Step 992 Action [[1.47710915]] Reward -7.442477138555716 Loss 0.0175363477319479\n",
      "Episode 4 Step 993 Action [[1.06641904]] Reward -5.802979365409331 Loss 0.012912897393107414\n",
      "Episode 4 Step 994 Action [[1.16696539]] Reward -4.4498269080894515 Loss 0.04809005558490753\n",
      "Episode 4 Step 995 Action [[1.34271258]] Reward -3.4844230881751828 Loss 0.07953223586082458\n",
      "Episode 4 Step 996 Action [[1.11363419]] Reward -2.886028836932099 Loss 0.03184667229652405\n",
      "Episode 4 Step 997 Action [[0.99009332]] Reward -2.569099092961249 Loss 0.08849382400512695\n",
      "Episode 4 Step 998 Action [[1.02858073]] Reward -2.5428252144147416 Loss 0.009324177168309689\n",
      "Episode 4 Step 999 Action [[1.23385921]] Reward -2.849476574572075 Loss 0.061267733573913574\n",
      "TOTAL REWARD @ 4-th Episode  : Reward -1182.8508910832777\n",
      "Total Step: 1000\n",
      "\n",
      "Episode : 5 Replay Buffer 1000\n",
      "[-0.79893217  0.60142115  0.50992453]\n",
      "Episode 5 Step 1000 Action [[1.21946053]] Reward -6.259072602261271 Loss 0.14615891873836517\n",
      "Episode 5 Step 1001 Action [[1.51080652]] Reward -6.653545675450172 Loss 0.16172264516353607\n",
      "Episode 5 Step 1002 Action [[1.86689904]] Reward -7.307283039666608 Loss 0.1298777014017105\n",
      "Episode 5 Step 1003 Action [[1.8230539]] Reward -8.23187397169605 Loss 0.02870572730898857\n",
      "Episode 5 Step 1004 Action [[1.76306164]] Reward -9.373170693144738 Loss 0.158382385969162\n",
      "Episode 5 Step 1005 Action [[1.79702495]] Reward -10.673023114993054 Loss 0.020599165931344032\n",
      "Episode 5 Step 1006 Action [[1.90474813]] Reward -10.47191910235451 Loss 0.14824722707271576\n",
      "Episode 5 Step 1007 Action [[1.83175607]] Reward -9.475777946921653 Loss 0.03870866820216179\n",
      "Episode 5 Step 1008 Action [[1.88147863]] Reward -8.430566044501012 Loss 0.04252371937036514\n",
      "Episode 5 Step 1009 Action [[1.98944047]] Reward -7.385996071820998 Loss 0.18298573791980743\n",
      "Episode 5 Step 1010 Action [[1.72868568]] Reward -6.383778214795837 Loss 0.09716866910457611\n",
      "Episode 5 Step 1011 Action [[1.76068417]] Reward -5.433485765582288 Loss 0.19098499417304993\n",
      "Episode 5 Step 1012 Action [[1.59460819]] Reward -4.596239226163266 Loss 0.05826904624700546\n",
      "Episode 5 Step 1013 Action [[1.64812677]] Reward -3.880992830767741 Loss 0.16468065977096558\n",
      "Episode 5 Step 1014 Action [[1.26400863]] Reward -3.312091023211322 Loss 0.07576186954975128\n",
      "Episode 5 Step 1015 Action [[1.34016273]] Reward -2.879492255829382 Loss 0.050888363271951675\n",
      "Episode 5 Step 1016 Action [[1.00930682]] Reward -2.6049974905325164 Loss 0.14768171310424805\n",
      "Episode 5 Step 1017 Action [[0.92307991]] Reward -2.4850416236707713 Loss 0.07615772634744644\n",
      "Episode 5 Step 1018 Action [[1.09437154]] Reward -2.5347017297140115 Loss 0.1836128532886505\n",
      "Episode 5 Step 1019 Action [[1.36130553]] Reward -2.750075241602531 Loss 0.045821741223335266\n",
      "Episode 5 Step 1020 Action [[1.40149152]] Reward -3.1175275756604424 Loss 0.04966854304075241\n",
      "Episode 5 Step 1021 Action [[1.17456043]] Reward -3.6448578855839364 Loss 0.08307330310344696\n",
      "Episode 5 Step 1022 Action [[1.17538646]] Reward -4.365442524850827 Loss 0.059413887560367584\n",
      "Episode 5 Step 1023 Action [[1.29460125]] Reward -5.271284285963887 Loss 0.11542025208473206\n",
      "Episode 5 Step 1024 Action [[1.16309293]] Reward -6.334735359724269 Loss 0.0345604382455349\n",
      "Episode 5 Step 1025 Action [[0.94644859]] Reward -7.557958262479807 Loss 0.07319678366184235\n",
      "Episode 5 Step 1026 Action [[0.98788215]] Reward -8.918958028034076 Loss 0.04799807071685791\n",
      "Episode 5 Step 1027 Action [[1.02042762]] Reward -10.323046272550789 Loss 0.05162232369184494\n",
      "Episode 5 Step 1028 Action [[1.08238026]] Reward -11.698051321991912 Loss 0.04865575209259987\n",
      "Episode 5 Step 1029 Action [[1.25502217]] Reward -10.605685517967569 Loss 0.07156321406364441\n",
      "Episode 5 Step 1030 Action [[1.41709155]] Reward -9.187785081257143 Loss 0.056289851665496826\n",
      "Episode 5 Step 1031 Action [[1.07381419]] Reward -7.877002417163773 Loss 0.1028999611735344\n",
      "Episode 5 Step 1032 Action [[0.82599466]] Reward -6.774469461822948 Loss 0.057229042053222656\n",
      "Episode 5 Step 1033 Action [[0.74111268]] Reward -5.895679608521461 Loss 0.03872373700141907\n",
      "Episode 5 Step 1034 Action [[0.34555067]] Reward -5.249159832641491 Loss 0.1074533462524414\n",
      "Episode 5 Step 1035 Action [[0.34971108]] Reward -4.848537329322131 Loss 0.041170574724674225\n",
      "Episode 5 Step 1036 Action [[0.15462802]] Reward -4.677843492745918 Loss 0.019030073657631874\n",
      "Episode 5 Step 1037 Action [[-0.11503482]] Reward -4.7355428548614125 Loss 0.05899595469236374\n",
      "Episode 5 Step 1038 Action [[0.14879704]] Reward -5.001150546837344 Loss 0.057081811130046844\n",
      "Episode 5 Step 1039 Action [[0.0485561]] Reward -5.490110257222666 Loss 0.06168355047702789\n",
      "Episode 5 Step 1040 Action [[0.18407514]] Reward -6.1818420430904455 Loss 0.04405584558844566\n",
      "Episode 5 Step 1041 Action [[0.46248585]] Reward -7.07685467368812 Loss 0.151431143283844\n",
      "Episode 5 Step 1042 Action [[0.63243268]] Reward -8.175561685763014 Loss 0.07914440333843231\n",
      "Episode 5 Step 1043 Action [[0.77298201]] Reward -9.440820843415684 Loss 0.07369472086429596\n",
      "Episode 5 Step 1044 Action [[0.79797315]] Reward -10.823090054681883 Loss 0.10884644836187363\n",
      "Episode 5 Step 1045 Action [[0.89643568]] Reward -10.798244109150103 Loss 0.0588371679186821\n",
      "Episode 5 Step 1046 Action [[0.88572494]] Reward -9.639663883943355 Loss 0.03900831192731857\n",
      "Episode 5 Step 1047 Action [[0.70634345]] Reward -8.469047069876696 Loss 0.02726001851260662\n",
      "Episode 5 Step 1048 Action [[0.99162611]] Reward -7.3213329191049095 Loss 0.21178419888019562\n",
      "Episode 5 Step 1049 Action [[1.05896001]] Reward -6.284351816141373 Loss 0.09568140655755997\n",
      "Episode 5 Step 1050 Action [[0.78223727]] Reward -5.367192108464918 Loss 0.024467820301651955\n",
      "Episode 5 Step 1051 Action [[0.65646681]] Reward -4.57986843957079 Loss 0.033322542905807495\n",
      "Episode 5 Step 1052 Action [[0.47330078]] Reward -3.9643256493376104 Loss 0.1429670751094818\n",
      "Episode 5 Step 1053 Action [[0.1321201]] Reward -3.5377406718026707 Loss 0.10198862850666046\n",
      "Episode 5 Step 1054 Action [[-0.07525955]] Reward -3.319486628440809 Loss 0.015275100246071815\n",
      "Episode 5 Step 1055 Action [[0.08516339]] Reward -3.336452232120156 Loss 0.15394970774650574\n",
      "Episode 5 Step 1056 Action [[0.65564944]] Reward -3.590622526478525 Loss 0.08636314421892166\n",
      "Episode 5 Step 1057 Action [[0.64169474]] Reward -4.039503024147799 Loss 0.07310542464256287\n",
      "Episode 5 Step 1058 Action [[0.84878874]] Reward -4.6922567047020936 Loss 0.021263953298330307\n",
      "Episode 5 Step 1059 Action [[0.54578399]] Reward -5.521007851492039 Loss 0.07309852540493011\n",
      "Episode 5 Step 1060 Action [[0.65621261]] Reward -6.562615873450629 Loss 0.02914777770638466\n",
      "Episode 5 Step 1061 Action [[0.44709156]] Reward -7.751372760354673 Loss 0.03155931085348129\n",
      "Episode 5 Step 1062 Action [[0.33513948]] Reward -9.083729426209702 Loss 0.034230686724185944\n",
      "Episode 5 Step 1063 Action [[0.31023446]] Reward -10.496524208934817 Loss 0.04232651740312576\n",
      "Episode 5 Step 1064 Action [[0.30385712]] Reward -11.619919306540824 Loss 0.025101108476519585\n",
      "Episode 5 Step 1065 Action [[0.202593]] Reward -10.273391753248656 Loss 0.062120016664266586\n",
      "Episode 5 Step 1066 Action [[0.30581059]] Reward -8.955880126632966 Loss 0.01938387006521225\n",
      "Episode 5 Step 1067 Action [[0.77330412]] Reward -7.701402822711017 Loss 0.01521032303571701\n",
      "Episode 5 Step 1068 Action [[0.9459067]] Reward -6.549390491269539 Loss 0.06829334795475006\n",
      "Episode 5 Step 1069 Action [[1.06598939]] Reward -5.599100532770253 Loss 0.02153468318283558\n",
      "Episode 5 Step 1070 Action [[1.29230766]] Reward -4.902078575201118 Loss 0.04422592371702194\n",
      "Episode 5 Step 1071 Action [[1.25865669]] Reward -4.490274931764354 Loss 0.032517656683921814\n",
      "Episode 5 Step 1072 Action [[1.04073864]] Reward -4.391252840521488 Loss 0.02311520092189312\n",
      "Episode 5 Step 1073 Action [[0.90179088]] Reward -4.5978640448472525 Loss 0.04052988439798355\n",
      "Episode 5 Step 1074 Action [[0.80650352]] Reward -5.094859449675015 Loss 0.06694919615983963\n",
      "Episode 5 Step 1075 Action [[1.03465454]] Reward -5.868318280034258 Loss 0.05682799965143204\n",
      "Episode 5 Step 1076 Action [[0.98319729]] Reward -6.932421664766222 Loss 0.13128012418746948\n",
      "Episode 5 Step 1077 Action [[0.82547719]] Reward -8.237757616913665 Loss 0.018602071329951286\n",
      "Episode 5 Step 1078 Action [[0.63420534]] Reward -9.709156358592205 Loss 0.03288039565086365\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 5 Step 1079 Action [[0.59296325]] Reward -11.256459336991169 Loss 0.1157202273607254\n",
      "Episode 5 Step 1080 Action [[0.55382922]] Reward -11.008593273908337 Loss 0.08406399935483932\n",
      "Episode 5 Step 1081 Action [[0.143808]] Reward -9.695932553578054 Loss 0.09567546099424362\n",
      "Episode 5 Step 1082 Action [[0.33172073]] Reward -8.353384181851816 Loss 0.0924496054649353\n",
      "Episode 5 Step 1083 Action [[0.29315954]] Reward -7.109038238931339 Loss 0.2747610807418823\n",
      "Episode 5 Step 1084 Action [[0.49278374]] Reward -5.988564182113564 Loss 0.05151975154876709\n",
      "Episode 5 Step 1085 Action [[0.65610686]] Reward -5.049521606348061 Loss 0.13116328418254852\n",
      "Episode 5 Step 1086 Action [[0.24683587]] Reward -4.301312549562897 Loss 0.2653486132621765\n",
      "Episode 5 Step 1087 Action [[0.23909917]] Reward -3.7328742995917175 Loss 0.04775514826178551\n",
      "Episode 5 Step 1088 Action [[0.43731264]] Reward -3.3840005632689434 Loss 0.2381487935781479\n",
      "Episode 5 Step 1089 Action [[0.70947013]] Reward -3.2521804345252527 Loss 0.06541052460670471\n",
      "Episode 5 Step 1090 Action [[0.8228288]] Reward -3.3179730523304323 Loss 0.08887318521738052\n",
      "Episode 5 Step 1091 Action [[0.74484497]] Reward -3.5697603930227237 Loss 0.09727787226438522\n",
      "Episode 5 Step 1092 Action [[0.504944]] Reward -4.0162484286604085 Loss 0.1461189091205597\n",
      "Episode 5 Step 1093 Action [[0.2761461]] Reward -4.682507108260531 Loss 0.08136183768510818\n",
      "Episode 5 Step 1094 Action [[0.23374549]] Reward -5.586178927745055 Loss 0.024698970839381218\n",
      "Episode 5 Step 1095 Action [[0.6668428]] Reward -6.70907909860368 Loss 0.0897420346736908\n",
      "Episode 5 Step 1096 Action [[0.76651287]] Reward -7.949673096010517 Loss 0.08876535296440125\n",
      "Episode 5 Step 1097 Action [[0.51861188]] Reward -9.27777779423675 Loss 0.051394157111644745\n",
      "Episode 5 Step 1098 Action [[0.65900472]] Reward -10.683814075763072 Loss 0.0557599738240242\n",
      "Episode 5 Step 1099 Action [[0.80467112]] Reward -11.462956923576039 Loss 0.05483053997159004\n",
      "Episode 5 Step 1100 Action [[0.74704087]] Reward -10.079765139875702 Loss 0.05825420469045639\n",
      "Episode 5 Step 1101 Action [[0.71687583]] Reward -8.754299898319532 Loss 0.05288401618599892\n",
      "Episode 5 Step 1102 Action [[0.73206742]] Reward -7.538466654605823 Loss 0.0327974408864975\n",
      "Episode 5 Step 1103 Action [[0.90881258]] Reward -6.481929311008785 Loss 0.032279398292303085\n",
      "Episode 5 Step 1104 Action [[0.62318123]] Reward -5.623408689115345 Loss 0.019920609891414642\n",
      "Episode 5 Step 1105 Action [[0.36510361]] Reward -5.0173797506223625 Loss 0.17619794607162476\n",
      "Episode 5 Step 1106 Action [[0.31920932]] Reward -4.656303032398356 Loss 0.05582324042916298\n",
      "Episode 5 Step 1107 Action [[0.25719293]] Reward -4.531580050704077 Loss 0.03069235011935234\n",
      "Episode 5 Step 1108 Action [[0.28656067]] Reward -4.643295696803453 Loss 0.03951017186045647\n",
      "Episode 5 Step 1109 Action [[0.44796486]] Reward -4.993110325994924 Loss 0.07819098979234695\n",
      "Episode 5 Step 1110 Action [[0.71657758]] Reward -5.59231436013127 Loss 0.07954786717891693\n",
      "Episode 5 Step 1111 Action [[1.08274433]] Reward -6.457632755447587 Loss 0.09391562640666962\n",
      "Episode 5 Step 1112 Action [[1.09680366]] Reward -7.604076583422294 Loss 0.10741925984621048\n",
      "Episode 5 Step 1113 Action [[1.30823442]] Reward -8.973929703377797 Loss 0.027111653238534927\n",
      "Episode 5 Step 1114 Action [[1.07252311]] Reward -10.539067848131184 Loss 0.053511250764131546\n",
      "Episode 5 Step 1115 Action [[1.12582692]] Reward -11.552794930356056 Loss 0.0585566870868206\n",
      "Episode 5 Step 1116 Action [[1.59863414]] Reward -10.307848465523563 Loss 0.10465118288993835\n",
      "Episode 5 Step 1117 Action [[1.46905608]] Reward -9.068992876467098 Loss 0.03048599883913994\n",
      "Episode 5 Step 1118 Action [[1.53882781]] Reward -7.809046094494835 Loss 0.06089570000767708\n",
      "Episode 5 Step 1119 Action [[1.29664912]] Reward -6.604547402180154 Loss 0.0190416369587183\n",
      "Episode 5 Step 1120 Action [[1.48820791]] Reward -5.477318702950824 Loss 0.022128574550151825\n",
      "Episode 5 Step 1121 Action [[1.30100515]] Reward -4.509248149661671 Loss 0.03791853412985802\n",
      "Episode 5 Step 1122 Action [[1.34431445]] Reward -3.6874028726809467 Loss 0.006444105412811041\n",
      "Episode 5 Step 1123 Action [[0.83206883]] Reward -3.038142802134108 Loss 0.05939576029777527\n",
      "Episode 5 Step 1124 Action [[0.65789192]] Reward -2.5393577552951276 Loss 0.005359422415494919\n",
      "Episode 5 Step 1125 Action [[0.53356389]] Reward -2.22095028427662 Loss 0.014785773120820522\n",
      "Episode 5 Step 1126 Action [[0.37492484]] Reward -2.0855762196807888 Loss 0.010345086455345154\n",
      "Episode 5 Step 1127 Action [[0.73208394]] Reward -2.1403701835267044 Loss 0.00796440988779068\n",
      "Episode 5 Step 1128 Action [[0.85521016]] Reward -2.37433060903797 Loss 0.01605459302663803\n",
      "Episode 5 Step 1129 Action [[0.95863173]] Reward -2.786652065585452 Loss 0.03441780433058739\n",
      "Episode 5 Step 1130 Action [[0.43442732]] Reward -3.3838506362628094 Loss 0.03629784286022186\n",
      "Episode 5 Step 1131 Action [[0.46889189]] Reward -4.245536266777728 Loss 0.01834164932370186\n",
      "Episode 5 Step 1132 Action [[0.70081533]] Reward -5.353144469023696 Loss 0.01497812569141388\n",
      "Episode 5 Step 1133 Action [[0.86232072]] Reward -6.6665930082762985 Loss 0.015881020575761795\n",
      "Episode 5 Step 1134 Action [[0.89381077]] Reward -8.145332217546322 Loss 0.00901804305613041\n",
      "Episode 5 Step 1135 Action [[0.87791]] Reward -9.739588234759184 Loss 0.013951470144093037\n",
      "Episode 5 Step 1136 Action [[1.10449741]] Reward -11.374197126810524 Loss 0.012424176558852196\n",
      "Episode 5 Step 1137 Action [[1.28496532]] Reward -12.004136539258026 Loss 0.010807272046804428\n",
      "Episode 5 Step 1138 Action [[1.35954271]] Reward -10.334161773415975 Loss 0.16121622920036316\n",
      "Episode 5 Step 1139 Action [[1.16725177]] Reward -8.73611867925725 Loss 0.010732137598097324\n",
      "Episode 5 Step 1140 Action [[1.40726174]] Reward -7.312987196153432 Loss 0.021002881228923798\n",
      "Episode 5 Step 1141 Action [[1.37020818]] Reward -6.086132903897487 Loss 0.03903615102171898\n",
      "Episode 5 Step 1142 Action [[1.61478109]] Reward -5.140073553194917 Loss 0.0073563167825341225\n",
      "Episode 5 Step 1143 Action [[1.29593059]] Reward -4.494923079094827 Loss 0.023199215531349182\n",
      "Episode 5 Step 1144 Action [[1.47547885]] Reward -4.189070957017367 Loss 0.012946153990924358\n",
      "Episode 5 Step 1145 Action [[1.32001893]] Reward -4.212624191177995 Loss 0.012431173585355282\n",
      "Episode 5 Step 1146 Action [[1.45240749]] Reward -4.569897283504252 Loss 0.012357819825410843\n",
      "Episode 5 Step 1147 Action [[1.16044239]] Reward -5.269746992056986 Loss 0.011877389624714851\n",
      "Episode 5 Step 1148 Action [[1.6013339]] Reward -6.273178763540846 Loss 0.019501902163028717\n",
      "Episode 5 Step 1149 Action [[1.42448647]] Reward -7.62764969875732 Loss 0.005218407604843378\n",
      "Episode 5 Step 1150 Action [[1.63906574]] Reward -9.234268701087279 Loss 0.014216111972928047\n",
      "Episode 5 Step 1151 Action [[1.20926144]] Reward -11.071696838773892 Loss 0.009947093203663826\n",
      "Episode 5 Step 1152 Action [[1.32374186]] Reward -11.921052389282314 Loss 0.0065188342705369\n",
      "Episode 5 Step 1153 Action [[1.23273631]] Reward -10.540958721661445 Loss 0.006004403345286846\n",
      "Episode 5 Step 1154 Action [[1.36670025]] Reward -9.103972599790277 Loss 0.10582870244979858\n",
      "Episode 5 Step 1155 Action [[1.50638609]] Reward -7.698471857681068 Loss 0.013114752247929573\n",
      "Episode 5 Step 1156 Action [[1.55131793]] Reward -6.382779817195993 Loss 0.00618261843919754\n",
      "Episode 5 Step 1157 Action [[1.36561565]] Reward -5.195003470688871 Loss 0.008491112850606441\n",
      "Episode 5 Step 1158 Action [[1.26204746]] Reward -4.150818197564657 Loss 0.008436517789959908\n",
      "Episode 5 Step 1159 Action [[1.35117237]] Reward -3.2835474221822585 Loss 0.011887900531291962\n",
      "Episode 5 Step 1160 Action [[1.35340381]] Reward -2.605932189013704 Loss 0.004577525891363621\n",
      "Episode 5 Step 1161 Action [[1.47889196]] Reward -2.093698293541519 Loss 0.005059016868472099\n",
      "Episode 5 Step 1162 Action [[1.30038348]] Reward -1.7292498242184067 Loss 0.0210347268730402\n",
      "Episode 5 Step 1163 Action [[1.08817915]] Reward -1.48372416712865 Loss 0.029551945626735687\n",
      "Episode 5 Step 1164 Action [[0.89869119]] Reward -1.3540686805200317 Loss 0.023981785401701927\n",
      "Episode 5 Step 1165 Action [[1.05312497]] Reward -1.3456708528054997 Loss 0.010599073022603989\n",
      "Episode 5 Step 1166 Action [[1.38052599]] Reward -1.4581109280495013 Loss 0.016534220427274704\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 5 Step 1167 Action [[1.17420999]] Reward -1.678516440733517 Loss 0.1568971425294876\n",
      "Episode 5 Step 1168 Action [[0.99152609]] Reward -2.038239015909531 Loss 0.03654269129037857\n",
      "Episode 5 Step 1169 Action [[1.05433602]] Reward -2.571583221586788 Loss 0.06692633777856827\n",
      "Episode 5 Step 1170 Action [[0.83399263]] Reward -3.2914538795234844 Loss 0.02586287260055542\n",
      "Episode 5 Step 1171 Action [[0.62655777]] Reward -4.255084260627006 Loss 0.044940683990716934\n",
      "Episode 5 Step 1172 Action [[0.81738353]] Reward -5.49672965336483 Loss 0.1826588362455368\n",
      "Episode 5 Step 1173 Action [[1.14843428]] Reward -6.96239028753422 Loss 0.12614627182483673\n",
      "Episode 5 Step 1174 Action [[0.68973754]] Reward -8.569311753998551 Loss 0.22477233409881592\n",
      "Episode 5 Step 1175 Action [[0.50701035]] Reward -10.38091127397687 Loss 0.05002836510539055\n",
      "Episode 5 Step 1176 Action [[0.45578713]] Reward -12.272114479834563 Loss 0.28498953580856323\n",
      "Episode 5 Step 1177 Action [[0.61912209]] Reward -12.25333795641817 Loss 0.07003559172153473\n",
      "Episode 5 Step 1178 Action [[0.52375694]] Reward -10.44322533381399 Loss 0.2544305920600891\n",
      "Episode 5 Step 1179 Action [[0.66214415]] Reward -8.696929514629476 Loss 0.15244799852371216\n",
      "Episode 5 Step 1180 Action [[0.68872466]] Reward -7.067729930544179 Loss 0.09334233403205872\n",
      "Episode 5 Step 1181 Action [[0.79659544]] Reward -5.651925442769626 Loss 0.29502078890800476\n",
      "Episode 5 Step 1182 Action [[0.82027932]] Reward -4.496445492155656 Loss 0.012243928387761116\n",
      "Episode 5 Step 1183 Action [[0.71736311]] Reward -3.6379882403198236 Loss 0.18880367279052734\n",
      "Episode 5 Step 1184 Action [[0.70473151]] Reward -3.0849287535840233 Loss 0.13373802602291107\n",
      "Episode 5 Step 1185 Action [[0.32362925]] Reward -2.822905932255085 Loss 0.05394718423485756\n",
      "Episode 5 Step 1186 Action [[0.37131825]] Reward -2.836504689306134 Loss 0.16072261333465576\n",
      "Episode 5 Step 1187 Action [[0.39309091]] Reward -3.1153077309628348 Loss 0.034572165459394455\n",
      "Episode 5 Step 1188 Action [[0.46655607]] Reward -3.671391991314744 Loss 0.11426697671413422\n",
      "Episode 5 Step 1189 Action [[0.31192749]] Reward -4.524247789404652 Loss 0.15368902683258057\n",
      "Episode 5 Step 1190 Action [[0.47073817]] Reward -5.657914104696828 Loss 0.03092901222407818\n",
      "Episode 5 Step 1191 Action [[0.48888047]] Reward -7.094503340842588 Loss 0.07233589142560959\n",
      "Episode 5 Step 1192 Action [[0.32360584]] Reward -8.785831256137376 Loss 0.08020082116127014\n",
      "Episode 5 Step 1193 Action [[0.55116926]] Reward -10.625224163258514 Loss 0.11889798939228058\n",
      "Episode 5 Step 1194 Action [[0.99030305]] Reward -12.583155304259707 Loss 0.03736409544944763\n",
      "Episode 5 Step 1195 Action [[1.19387552]] Reward -11.632665745533684 Loss 0.04118037968873978\n",
      "Episode 5 Step 1196 Action [[1.23475892]] Reward -10.072793937902398 Loss 0.023673824965953827\n",
      "Episode 5 Step 1197 Action [[1.04831602]] Reward -8.504597895666643 Loss 0.008059407584369183\n",
      "Episode 5 Step 1198 Action [[0.80473139]] Reward -6.974719724247869 Loss 0.04630883038043976\n",
      "Episode 5 Step 1199 Action [[0.78566485]] Reward -5.560664918521244 Loss 0.009840408340096474\n",
      "TOTAL REWARD @ 5-th Episode  : Reward -1278.8747388910288\n",
      "Total Step: 1200\n",
      "\n",
      "Episode : 6 Replay Buffer 1200\n",
      "[ 0.85323805  0.52152165 -0.97412472]\n",
      "Episode 6 Step 1200 Action [[0.77430206]] Reward -0.39649000441699095 Loss 0.04587147384881973\n",
      "Episode 6 Step 1201 Action [[0.59428963]] Reward -0.29807807775556333 Loss 0.01662282459437847\n",
      "Episode 6 Step 1202 Action [[0.40216711]] Reward -0.27600928111439454 Loss 0.008296051062643528\n",
      "Episode 6 Step 1203 Action [[0.67378515]] Reward -0.3185118573844219 Loss 0.015407823026180267\n",
      "Episode 6 Step 1204 Action [[0.88607707]] Reward -0.4384563337778898 Loss 0.015887729823589325\n",
      "Episode 6 Step 1205 Action [[0.82777243]] Reward -0.6642573798658671 Loss 0.009847365319728851\n",
      "Episode 6 Step 1206 Action [[0.91013369]] Reward -1.0214258793984814 Loss 0.005690613761544228\n",
      "Episode 6 Step 1207 Action [[0.95861874]] Reward -1.5657553256395804 Loss 0.009262127801775932\n",
      "Episode 6 Step 1208 Action [[1.03532448]] Reward -2.363746899604272 Loss 0.01956147700548172\n",
      "Episode 6 Step 1209 Action [[0.89688113]] Reward -3.502676238369389 Loss 0.01034587249159813\n",
      "Episode 6 Step 1210 Action [[0.75818593]] Reward -5.037412511549215 Loss 0.015456557273864746\n",
      "Episode 6 Step 1211 Action [[1.06594653]] Reward -7.0182381073709905 Loss 0.02247488498687744\n",
      "Episode 6 Step 1212 Action [[1.28137009]] Reward -9.542961542517856 Loss 0.006741771474480629\n",
      "Episode 6 Step 1213 Action [[1.52214469]] Reward -12.541651048658503 Loss 0.0102181201800704\n",
      "Episode 6 Step 1214 Action [[1.32389356]] Reward -15.359833480038791 Loss 0.014210150577127934\n",
      "Episode 6 Step 1215 Action [[1.31393852]] Reward -14.665622217076274 Loss 0.12007981538772583\n",
      "Episode 6 Step 1216 Action [[1.33284846]] Reward -12.524950862243244 Loss 0.013036207295954227\n",
      "Episode 6 Step 1217 Action [[1.33103955]] Reward -10.344910948385717 Loss 0.00650986609980464\n",
      "Episode 6 Step 1218 Action [[1.69607592]] Reward -8.278720984336273 Loss 0.03935111314058304\n",
      "Episode 6 Step 1219 Action [[1.46638283]] Reward -6.536504299545017 Loss 0.006512770429253578\n",
      "Episode 6 Step 1220 Action [[1.56158655]] Reward -5.090317745363644 Loss 0.007216854486614466\n",
      "Episode 6 Step 1221 Action [[1.58215448]] Reward -4.03540763192486 Loss 0.004614425823092461\n",
      "Episode 6 Step 1222 Action [[1.10751318]] Reward -3.34917533216576 Loss 0.18378810584545135\n",
      "Episode 6 Step 1223 Action [[1.24283621]] Reward -2.942704186872915 Loss 0.014973435550928116\n",
      "Episode 6 Step 1224 Action [[1.21871594]] Reward -2.9076465386948893 Loss 0.036580510437488556\n",
      "Episode 6 Step 1225 Action [[1.19644645]] Reward -3.241318156812401 Loss 0.017300011590123177\n",
      "Episode 6 Step 1226 Action [[0.90994324]] Reward -4.00543027775112 Loss 0.014226112514734268\n",
      "Episode 6 Step 1227 Action [[0.79374627]] Reward -5.243923766335041 Loss 0.019437164068222046\n",
      "Episode 6 Step 1228 Action [[0.64447505]] Reward -7.086291356519904 Loss 0.04645209014415741\n",
      "Episode 6 Step 1229 Action [[0.48203568]] Reward -9.367391718505045 Loss 0.015174109488725662\n",
      "Episode 6 Step 1230 Action [[0.42115536]] Reward -11.079736098912656 Loss 0.019908901304006577\n",
      "Episode 6 Step 1231 Action [[0.23243928]] Reward -13.147044031956511 Loss 0.13548320531845093\n",
      "Episode 6 Step 1232 Action [[0.28998602]] Reward -15.512343034945578 Loss 0.01182763185352087\n",
      "Episode 6 Step 1233 Action [[0.17997856]] Reward -14.566824006782124 Loss 0.004692293703556061\n",
      "Episode 6 Step 1234 Action [[0.60880429]] Reward -12.196523094640488 Loss 0.2119525671005249\n",
      "Episode 6 Step 1235 Action [[0.69734528]] Reward -9.92424877910286 Loss 0.010878914967179298\n",
      "Episode 6 Step 1236 Action [[0.97417742]] Reward -7.818251504461704 Loss 0.05286390334367752\n",
      "Episode 6 Step 1237 Action [[0.95761371]] Reward -6.033243431939447 Loss 0.004778559319674969\n",
      "Episode 6 Step 1238 Action [[1.13373188]] Reward -4.5766820804289 Loss 0.02049810066819191\n",
      "Episode 6 Step 1239 Action [[0.87830832]] Reward -3.4864097186102785 Loss 0.03159487992525101\n",
      "Episode 6 Step 1240 Action [[0.57536028]] Reward -2.672288162117464 Loss 0.016369827091693878\n",
      "Episode 6 Step 1241 Action [[0.84228854]] Reward -2.098305819541215 Loss 0.14033521711826324\n",
      "Episode 6 Step 1242 Action [[0.76923588]] Reward -1.7978946387605277 Loss 0.01793733611702919\n",
      "Episode 6 Step 1243 Action [[0.79233405]] Reward -1.7034129155988595 Loss 0.010354056023061275\n",
      "Episode 6 Step 1244 Action [[0.78137589]] Reward -1.8266751604382276 Loss 0.03032202273607254\n",
      "Episode 6 Step 1245 Action [[1.12697051]] Reward -2.1906479760568556 Loss 0.3176267445087433\n",
      "Episode 6 Step 1246 Action [[0.69519423]] Reward -2.9043981450560903 Loss 0.2218567430973053\n",
      "Episode 6 Step 1247 Action [[0.74020626]] Reward -3.9427561781918725 Loss 0.047807637602090836\n",
      "Episode 6 Step 1248 Action [[0.97878343]] Reward -5.472692351074457 Loss 0.06121760606765747\n",
      "Episode 6 Step 1249 Action [[0.97146741]] Reward -7.636574480392632 Loss 0.14104345440864563\n",
      "Episode 6 Step 1250 Action [[1.10799098]] Reward -10.43495446825935 Loss 0.06673154979944229\n",
      "Episode 6 Step 1251 Action [[1.12372614]] Reward -12.407144642189815 Loss 0.08034321665763855\n",
      "Episode 6 Step 1252 Action [[1.07682292]] Reward -14.712930813263416 Loss 0.014015300199389458\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 6 Step 1253 Action [[0.878373]] Reward -15.296112838022996 Loss 0.0751212015748024\n",
      "Episode 6 Step 1254 Action [[1.14951127]] Reward -13.067315723454913 Loss 0.009582008235156536\n",
      "Episode 6 Step 1255 Action [[1.04587778]] Reward -10.854821808945943 Loss 0.037511084228754044\n",
      "Episode 6 Step 1256 Action [[0.95307751]] Reward -8.699805270864095 Loss 0.0774265006184578\n",
      "Episode 6 Step 1257 Action [[1.11891962]] Reward -6.763292976474726 Loss 0.020619451999664307\n",
      "Episode 6 Step 1258 Action [[1.08077843]] Reward -5.1924749616946295 Loss 0.025273190811276436\n",
      "Episode 6 Step 1259 Action [[0.83344148]] Reward -3.978927983438224 Loss 0.012153632938861847\n",
      "Episode 6 Step 1260 Action [[1.00243081]] Reward -3.0812154742374123 Loss 0.01552811823785305\n",
      "Episode 6 Step 1261 Action [[1.15491909]] Reward -2.5281468773477482 Loss 0.03388771414756775\n",
      "Episode 6 Step 1262 Action [[1.32833383]] Reward -2.2816610358197607 Loss 0.018164603039622307\n",
      "Episode 6 Step 1263 Action [[1.1600057]] Reward -2.3363654403928953 Loss 0.04218466952443123\n",
      "Episode 6 Step 1264 Action [[0.95811902]] Reward -2.6709001258894562 Loss 0.04461902379989624\n",
      "Episode 6 Step 1265 Action [[1.04908466]] Reward -3.334440432835748 Loss 0.03149725869297981\n",
      "Episode 6 Step 1266 Action [[0.68899305]] Reward -4.459613468802085 Loss 0.12070564180612564\n",
      "Episode 6 Step 1267 Action [[0.59646485]] Reward -6.067084152217059 Loss 0.08797843754291534\n",
      "Episode 6 Step 1268 Action [[0.60399397]] Reward -8.282786543697823 Loss 0.09473911672830582\n",
      "Episode 6 Step 1269 Action [[0.18686998]] Reward -10.54945157349056 Loss 0.1557478904724121\n",
      "Episode 6 Step 1270 Action [[0.3533092]] Reward -12.510469156360584 Loss 0.058216262608766556\n",
      "Episode 6 Step 1271 Action [[0.34587254]] Reward -14.797587607036156 Loss 0.13419616222381592\n",
      "Episode 6 Step 1272 Action [[0.14382987]] Reward -15.24292564481326 Loss 0.07997757196426392\n",
      "Episode 6 Step 1273 Action [[-0.30213777]] Reward -12.885836469549124 Loss 0.3573671281337738\n",
      "Episode 6 Step 1274 Action [[-0.60830763]] Reward -10.419818544752557 Loss 0.07335945963859558\n",
      "Episode 6 Step 1275 Action [[-0.59130277]] Reward -8.05619770902111 Loss 0.03242766857147217\n",
      "Episode 6 Step 1276 Action [[-0.98770569]] Reward -6.009750596699948 Loss 0.041946519166231155\n",
      "Episode 6 Step 1277 Action [[-0.72834122]] Reward -4.291209973607128 Loss 0.15611523389816284\n",
      "Episode 6 Step 1278 Action [[-0.63406187]] Reward -3.0051909091658864 Loss 0.3220392167568207\n",
      "Episode 6 Step 1279 Action [[-0.56669482]] Reward -2.065319710282586 Loss 0.22005599737167358\n",
      "Episode 6 Step 1280 Action [[-0.47366027]] Reward -1.3982887697552577 Loss 0.13764885067939758\n",
      "Episode 6 Step 1281 Action [[-0.14468656]] Reward -0.9392969003153544 Loss 0.07956104725599289\n",
      "Episode 6 Step 1282 Action [[0.04732291]] Reward -0.6419802662900248 Loss 0.10823211073875427\n",
      "Episode 6 Step 1283 Action [[0.03095171]] Reward -0.44931454227094086 Loss 0.13147513568401337\n",
      "Episode 6 Step 1284 Action [[0.42633531]] Reward -0.32478067493240825 Loss 0.05768245831131935\n",
      "Episode 6 Step 1285 Action [[0.81061561]] Reward -0.2564082160729373 Loss 0.08279551565647125\n",
      "Episode 6 Step 1286 Action [[1.12471184]] Reward -0.22183132457643093 Loss 0.2559954524040222\n",
      "Episode 6 Step 1287 Action [[1.37113704]] Reward -0.206476766041076 Loss 0.04767106473445892\n",
      "Episode 6 Step 1288 Action [[1.27365998]] Reward -0.20126136341259904 Loss 0.10203070938587189\n",
      "Episode 6 Step 1289 Action [[1.36576598]] Reward -0.20557124562227932 Loss 0.04099911078810692\n",
      "Episode 6 Step 1290 Action [[1.74967772]] Reward -0.2195070614967791 Loss 0.024021459743380547\n",
      "Episode 6 Step 1291 Action [[1.64990251]] Reward -0.2353950697822544 Loss 0.044102881103754044\n",
      "Episode 6 Step 1292 Action [[1.74795068]] Reward -0.259737411763803 Loss 0.04904255270957947\n",
      "Episode 6 Step 1293 Action [[1.30905387]] Reward -0.2894385782956602 Loss 0.026252921670675278\n",
      "Episode 6 Step 1294 Action [[1.21632281]] Reward -0.3421308893115346 Loss 0.020145639777183533\n",
      "Episode 6 Step 1295 Action [[1.31268957]] Reward -0.42124255108863623 Loss 0.03258177638053894\n",
      "Episode 6 Step 1296 Action [[1.47696418]] Reward -0.5294511837021788 Loss 0.03221370652318001\n",
      "Episode 6 Step 1297 Action [[1.52788453]] Reward -0.6708235238408913 Loss 0.010723039507865906\n",
      "Episode 6 Step 1298 Action [[1.02892694]] Reward -0.8589817290618982 Loss 0.017707310616970062\n",
      "Episode 6 Step 1299 Action [[1.1746862]] Reward -1.1508976877291932 Loss 0.018513048067688942\n",
      "Episode 6 Step 1300 Action [[1.19407286]] Reward -1.5415982064625933 Loss 0.017030242830514908\n",
      "Episode 6 Step 1301 Action [[1.15289131]] Reward -2.0694029589297323 Loss 0.020754419267177582\n",
      "Episode 6 Step 1302 Action [[0.87599984]] Reward -2.778455260969863 Loss 0.012007789686322212\n",
      "Episode 6 Step 1303 Action [[0.84583286]] Reward -3.7457750719730982 Loss 0.012241417542099953\n",
      "Episode 6 Step 1304 Action [[1.14149197]] Reward -4.991358663419843 Loss 0.06385213136672974\n",
      "Episode 6 Step 1305 Action [[1.51564058]] Reward -6.475930580756771 Loss 0.011993108317255974\n",
      "Episode 6 Step 1306 Action [[1.47229183]] Reward -8.13584666205596 Loss 0.024875473231077194\n",
      "Episode 6 Step 1307 Action [[1.80236083]] Reward -9.965404885485624 Loss 0.021944498643279076\n",
      "Episode 6 Step 1308 Action [[1.90217785]] Reward -11.776316054466264 Loss 0.04423075541853905\n",
      "Episode 6 Step 1309 Action [[1.57885776]] Reward -13.407032724144814 Loss 0.034959807991981506\n",
      "Episode 6 Step 1310 Action [[1.49247089]] Reward -11.407501908497292 Loss 0.010955189354717731\n",
      "Episode 6 Step 1311 Action [[1.19068612]] Reward -9.482179285840555 Loss 0.014454271644353867\n",
      "Episode 6 Step 1312 Action [[1.01428427]] Reward -7.741260328108419 Loss 0.20958781242370605\n",
      "Episode 6 Step 1313 Action [[1.34770727]] Reward -6.2343913027400175 Loss 0.019959706813097\n",
      "Episode 6 Step 1314 Action [[0.96943112]] Reward -4.970341821203126 Loss 0.03560029715299606\n",
      "Episode 6 Step 1315 Action [[0.79777187]] Reward -4.04985415233711 Loss 0.004359998274594545\n",
      "Episode 6 Step 1316 Action [[0.89331681]] Reward -3.4427920460433703 Loss 0.33198845386505127\n",
      "Episode 6 Step 1317 Action [[1.24146638]] Reward -3.1344963441210414 Loss 0.040983811020851135\n",
      "Episode 6 Step 1318 Action [[0.68923877]] Reward -3.140476265013727 Loss 0.06328174471855164\n",
      "Episode 6 Step 1319 Action [[0.47741641]] Reward -3.450079882194076 Loss 0.4747465252876282\n",
      "Episode 6 Step 1320 Action [[1.22879622]] Reward -4.043639889033527 Loss 0.1191563606262207\n",
      "Episode 6 Step 1321 Action [[1.03179324]] Reward -5.011778616290017 Loss 0.05663164332509041\n",
      "Episode 6 Step 1322 Action [[0.9264201]] Reward -6.306003841997646 Loss 0.16208720207214355\n",
      "Episode 6 Step 1323 Action [[1.13900334]] Reward -7.898439462734605 Loss 0.032743439078330994\n",
      "Episode 6 Step 1324 Action [[1.39954093]] Reward -9.782655215756972 Loss 0.13444861769676208\n",
      "Episode 6 Step 1325 Action [[1.35463019]] Reward -11.898429151748013 Loss 0.2868119776248932\n",
      "Episode 6 Step 1326 Action [[1.26675868]] Reward -12.47692579086689 Loss 0.08356784284114838\n",
      "Episode 6 Step 1327 Action [[1.33377013]] Reward -10.891365728343347 Loss 0.10746289044618607\n",
      "Episode 6 Step 1328 Action [[1.56164485]] Reward -9.269830854294112 Loss 0.025656282901763916\n",
      "Episode 6 Step 1329 Action [[1.56912448]] Reward -7.7065897334600555 Loss 0.08084948360919952\n",
      "Episode 6 Step 1330 Action [[1.55647906]] Reward -6.243077717853325 Loss 0.03952281177043915\n",
      "Episode 6 Step 1331 Action [[1.44458477]] Reward -4.9409529509395 Loss 0.2568836808204651\n",
      "Episode 6 Step 1332 Action [[1.37797826]] Reward -3.8270067292361705 Loss 0.053212955594062805\n",
      "Episode 6 Step 1333 Action [[1.12906094]] Reward -2.917394496783637 Loss 0.021644189953804016\n",
      "Episode 6 Step 1334 Action [[1.10162627]] Reward -2.1862857188688816 Loss 0.02053825929760933\n",
      "Episode 6 Step 1335 Action [[0.92034014]] Reward -1.6307532344464575 Loss 0.06773173063993454\n",
      "Episode 6 Step 1336 Action [[0.54428118]] Reward -1.2104061645706052 Loss 0.05749356374144554\n",
      "Episode 6 Step 1337 Action [[0.50322198]] Reward -0.8958438000455793 Loss 0.01879647746682167\n",
      "Episode 6 Step 1338 Action [[0.4124931]] Reward -0.6847867781843819 Loss 0.18651682138442993\n",
      "Episode 6 Step 1339 Action [[0.42404859]] Reward -0.5569865628984152 Loss 0.04781446233391762\n",
      "Episode 6 Step 1340 Action [[0.56761331]] Reward -0.5024846367576344 Loss 0.09317517280578613\n",
      "Episode 6 Step 1341 Action [[1.01993232]] Reward -0.5127736336378023 Loss 0.04334817826747894\n",
      "Episode 6 Step 1342 Action [[1.11805827]] Reward -0.5727006318947037 Loss 0.03301307559013367\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 6 Step 1343 Action [[0.70903803]] Reward -0.68175170063662 Loss 0.034981146454811096\n",
      "Episode 6 Step 1344 Action [[0.85491541]] Reward -0.8738953567046388 Loss 0.04042582958936691\n",
      "Episode 6 Step 1345 Action [[0.66944309]] Reward -1.1524538594308344 Loss 0.011592860333621502\n",
      "Episode 6 Step 1346 Action [[0.79553642]] Reward -1.5641518533760395 Loss 0.05744192749261856\n",
      "Episode 6 Step 1347 Action [[1.2288178]] Reward -2.1265798443417108 Loss 0.032942503690719604\n",
      "Episode 6 Step 1348 Action [[0.99627354]] Reward -2.8350847635515692 Loss 0.025893863290548325\n",
      "Episode 6 Step 1349 Action [[0.92165497]] Reward -3.7903225928218176 Loss 0.026970338076353073\n",
      "Episode 6 Step 1350 Action [[0.66705637]] Reward -5.018553757829585 Loss 0.04331362619996071\n",
      "Episode 6 Step 1351 Action [[1.08498156]] Reward -6.57451141298035 Loss 0.03658374398946762\n",
      "Episode 6 Step 1352 Action [[1.41526436]] Reward -8.322897515976209 Loss 0.007658867165446281\n",
      "Episode 6 Step 1353 Action [[1.57764943]] Reward -10.176592938123926 Loss 0.02645450085401535\n",
      "Episode 6 Step 1354 Action [[1.59325349]] Reward -12.040227782905593 Loss 0.025257056578993797\n",
      "Episode 6 Step 1355 Action [[1.9883853]] Reward -13.260947097968257 Loss 0.04718886315822601\n",
      "Episode 6 Step 1356 Action [[1.71735551]] Reward -11.17886287819445 Loss 0.00906605739146471\n",
      "Episode 6 Step 1357 Action [[1.56287778]] Reward -9.23033942228265 Loss 0.026952220126986504\n",
      "Episode 6 Step 1358 Action [[1.54657841]] Reward -7.478409959237684 Loss 0.312112957239151\n",
      "Episode 6 Step 1359 Action [[1.1635878]] Reward -5.981625029310135 Loss 0.04566628858447075\n",
      "Episode 6 Step 1360 Action [[1.07468238]] Reward -4.820755919595082 Loss 0.1462591141462326\n",
      "Episode 6 Step 1361 Action [[0.9862241]] Reward -3.9760238881684518 Loss 0.02804269827902317\n",
      "Episode 6 Step 1362 Action [[0.72216315]] Reward -3.4485367049889826 Loss 0.1976308524608612\n",
      "Episode 6 Step 1363 Action [[0.4238603]] Reward -3.2285324032760108 Loss 0.033086083829402924\n",
      "Episode 6 Step 1364 Action [[0.21380299]] Reward -3.2887858519120035 Loss 0.1322026401758194\n",
      "Episode 6 Step 1365 Action [[0.18463625]] Reward -3.6089521470621526 Loss 0.02308034524321556\n",
      "Episode 6 Step 1366 Action [[0.44989804]] Reward -4.1901983502719755 Loss 0.08420178294181824\n",
      "Episode 6 Step 1367 Action [[0.3494131]] Reward -5.0661007847342745 Loss 0.3942662179470062\n",
      "Episode 6 Step 1368 Action [[0.54552521]] Reward -6.211116949945582 Loss 0.06614600121974945\n",
      "Episode 6 Step 1369 Action [[0.71146825]] Reward -7.638819604468541 Loss 0.3151897192001343\n",
      "Episode 6 Step 1370 Action [[0.49856608]] Reward -9.313699780353396 Loss 0.027457600459456444\n",
      "Episode 6 Step 1371 Action [[0.34184006]] Reward -11.100504174099937 Loss 0.16729113459587097\n",
      "Episode 6 Step 1372 Action [[0.20609672]] Reward -12.226122441239186 Loss 0.33799630403518677\n",
      "Episode 6 Step 1373 Action [[0.02556133]] Reward -10.684854198399783 Loss 0.18286782503128052\n",
      "Episode 6 Step 1374 Action [[-0.09101048]] Reward -9.11602202916304 Loss 0.5657333731651306\n",
      "Episode 6 Step 1375 Action [[-0.15204585]] Reward -7.604933666630928 Loss 0.06715761125087738\n",
      "Episode 6 Step 1376 Action [[-0.39404491]] Reward -6.234402600250354 Loss 0.3682236671447754\n",
      "Episode 6 Step 1377 Action [[-0.2478566]] Reward -5.051551771123629 Loss 0.91568922996521\n",
      "Episode 6 Step 1378 Action [[0.01508933]] Reward -4.130490035608343 Loss 0.33442622423171997\n",
      "Episode 6 Step 1379 Action [[0.13244479]] Reward -3.4745693268133446 Loss 1.3068358898162842\n",
      "Episode 6 Step 1380 Action [[0.14076846]] Reward -3.0598905879428098 Loss 0.7912060022354126\n",
      "Episode 6 Step 1381 Action [[0.53430463]] Reward -2.874558076937671 Loss 0.14578096568584442\n",
      "Episode 6 Step 1382 Action [[0.95396034]] Reward -2.9035426172947467 Loss 1.4291789531707764\n",
      "Episode 6 Step 1383 Action [[1.13316822]] Reward -3.1135271984569246 Loss 0.1878836750984192\n",
      "Episode 6 Step 1384 Action [[1.34712271]] Reward -3.4926849621695863 Loss 0.9243045449256897\n",
      "Episode 6 Step 1385 Action [[1.0220165]] Reward -4.0281361190628395 Loss 1.0974446535110474\n",
      "Episode 6 Step 1386 Action [[0.94063761]] Reward -4.76393993889904 Loss 0.29032182693481445\n",
      "Episode 6 Step 1387 Action [[0.7853943]] Reward -5.692194960841475 Loss 0.4848926067352295\n",
      "Episode 6 Step 1388 Action [[0.85929476]] Reward -6.813980776129773 Loss 1.636987328529358\n",
      "Episode 6 Step 1389 Action [[1.17119667]] Reward -8.072175091173866 Loss 0.24763131141662598\n",
      "Episode 6 Step 1390 Action [[1.25066817]] Reward -9.368012417263596 Loss 0.7967204451560974\n",
      "Episode 6 Step 1391 Action [[1.4078184]] Reward -10.658625887934777 Loss 1.112101435661316\n",
      "Episode 6 Step 1392 Action [[1.57985712]] Reward -11.326318982785093 Loss 0.7322505116462708\n",
      "Episode 6 Step 1393 Action [[1.35243307]] Reward -9.945208183765923 Loss 1.3443381786346436\n",
      "Episode 6 Step 1394 Action [[1.23678121]] Reward -8.679192100960883 Loss 1.6293296813964844\n",
      "Episode 6 Step 1395 Action [[0.99243754]] Reward -7.561933287159426 Loss 0.3721665143966675\n",
      "Episode 6 Step 1396 Action [[1.00938569]] Reward -6.6403435896237735 Loss 1.5182658433914185\n",
      "Episode 6 Step 1397 Action [[1.17588403]] Reward -5.927676225551592 Loss 1.892978310585022\n",
      "Episode 6 Step 1398 Action [[1.01416761]] Reward -5.452608118188419 Loss 0.41232895851135254\n",
      "Episode 6 Step 1399 Action [[0.92260581]] Reward -5.2430041267066745 Loss 2.287534475326538\n",
      "Now we save model\n",
      "TOTAL REWARD @ 6-th Episode  : Reward -1122.8866796750242\n",
      "Total Step: 1400\n",
      "\n",
      "Episode : 7 Replay Buffer 1400\n",
      "[ 0.21485086  0.97664687 -0.58522649]\n",
      "Episode 7 Step 1400 Action [[0.54241534]] Reward -1.8685557898687917 Loss 0.7359611988067627\n",
      "Episode 7 Step 1401 Action [[-0.04420734]] Reward -1.870333107605626 Loss 0.22764308750629425\n",
      "Episode 7 Step 1402 Action [[-0.27914546]] Reward -2.0895100532821322 Loss 0.8800948858261108\n",
      "Episode 7 Step 1403 Action [[-0.35112957]] Reward -2.512861428615691 Loss 0.5127929449081421\n",
      "Episode 7 Step 1404 Action [[-0.840965]] Reward -3.1574180683125292 Loss 0.22110925614833832\n",
      "Episode 7 Step 1405 Action [[-0.47115438]] Reward -3.9906703092650835 Loss 1.0161312818527222\n",
      "Episode 7 Step 1406 Action [[-0.5183513]] Reward -5.105344936608853 Loss 0.38064634799957275\n",
      "Episode 7 Step 1407 Action [[-0.63902242]] Reward -6.4692239677405015 Loss 0.33376264572143555\n",
      "Episode 7 Step 1408 Action [[-0.62098145]] Reward -8.03800731867677 Loss 0.4945020079612732\n",
      "Episode 7 Step 1409 Action [[-0.49855283]] Reward -9.767656361383919 Loss 0.42276090383529663\n",
      "Episode 7 Step 1410 Action [[-0.74382614]] Reward -11.591882273597925 Loss 0.1835271567106247\n",
      "Episode 7 Step 1411 Action [[-0.83412245]] Reward -12.514004220103905 Loss 0.26865100860595703\n",
      "Episode 7 Step 1412 Action [[-0.56237886]] Reward -10.750256510535905 Loss 0.33023491501808167\n",
      "Episode 7 Step 1413 Action [[-0.50731015]] Reward -9.061662484603971 Loss 0.16359618306159973\n",
      "Episode 7 Step 1414 Action [[-0.758554]] Reward -7.487057710412184 Loss 0.1358267366886139\n",
      "Episode 7 Step 1415 Action [[-0.82127724]] Reward -6.069520720594 Loss 0.26505959033966064\n",
      "Episode 7 Step 1416 Action [[-0.54744312]] Reward -4.898759128310201 Loss 0.1948719024658203\n",
      "Episode 7 Step 1417 Action [[-0.41790288]] Reward -4.02665774800696 Loss 0.0912865474820137\n",
      "Episode 7 Step 1418 Action [[-0.40106036]] Reward -3.4351656390817316 Loss 0.11561702191829681\n",
      "Episode 7 Step 1419 Action [[-0.43331114]] Reward -3.1129113946924383 Loss 0.27456367015838623\n",
      "Episode 7 Step 1420 Action [[0.1336537]] Reward -3.058103840951055 Loss 0.056980181485414505\n",
      "Episode 7 Step 1421 Action [[0.59708007]] Reward -3.24638473744838 Loss 0.1584159880876541\n",
      "Episode 7 Step 1422 Action [[0.74776787]] Reward -3.637346559333173 Loss 0.19222231209278107\n",
      "Episode 7 Step 1423 Action [[0.77466527]] Reward -4.224121002633005 Loss 0.10672590881586075\n",
      "Episode 7 Step 1424 Action [[0.95501362]] Reward -5.009967384626293 Loss 0.08142340183258057\n",
      "Episode 7 Step 1425 Action [[1.19763496]] Reward -5.9667248049218875 Loss 0.25519001483917236\n",
      "Episode 7 Step 1426 Action [[1.29473471]] Reward -7.049692414214918 Loss 0.09378766268491745\n",
      "Episode 7 Step 1427 Action [[0.94619747]] Reward -8.224961973333665 Loss 0.06171555072069168\n",
      "Episode 7 Step 1428 Action [[0.87931814]] Reward -9.50865090123503 Loss 0.166304349899292\n",
      "Episode 7 Step 1429 Action [[1.02919725]] Reward -10.813575747492454 Loss 0.1788634955883026\n",
      "Episode 7 Step 1430 Action [[1.3029904]] Reward -11.073226592547133 Loss 0.04875960201025009\n",
      "Episode 7 Step 1431 Action [[1.55149064]] Reward -9.730852119562165 Loss 0.12302803993225098\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 7 Step 1432 Action [[1.23591002]] Reward -8.464514188265932 Loss 0.23928043246269226\n",
      "Episode 7 Step 1433 Action [[1.33195453]] Reward -7.38067270058706 Loss 0.19341185688972473\n",
      "Episode 7 Step 1434 Action [[1.44154346]] Reward -6.487854099968948 Loss 0.1052008718252182\n",
      "Episode 7 Step 1435 Action [[1.75648715]] Reward -5.833626530766564 Loss 0.15039151906967163\n",
      "Episode 7 Step 1436 Action [[1.72243216]] Reward -5.4554187738463895 Loss 0.10504207015037537\n",
      "Episode 7 Step 1437 Action [[1.56354375]] Reward -5.390865378896815 Loss 0.029989566653966904\n",
      "Episode 7 Step 1438 Action [[1.09937661]] Reward -5.635327609903144 Loss 0.06374450027942657\n",
      "Episode 7 Step 1439 Action [[0.49948648]] Reward -6.145802781811564 Loss 0.16531303524971008\n",
      "Episode 7 Step 1440 Action [[0.57387275]] Reward -6.850983423782502 Loss 0.09252506494522095\n",
      "Episode 7 Step 1441 Action [[-0.159596]] Reward -7.7502302219790185 Loss 0.08959140628576279\n",
      "Episode 7 Step 1442 Action [[0.36677008]] Reward -8.710757290384741 Loss 0.1028873398900032\n",
      "Episode 7 Step 1443 Action [[-0.01764206]] Reward -9.80912940048169 Loss 0.080975741147995\n",
      "Episode 7 Step 1444 Action [[-0.03624608]] Reward -10.899011657327788 Loss 0.06307098269462585\n",
      "Episode 7 Step 1445 Action [[-0.05066064]] Reward -10.009045812419675 Loss 0.057805389165878296\n",
      "Episode 7 Step 1446 Action [[-0.1716205]] Reward -9.004294887976776 Loss 0.14000341296195984\n",
      "Episode 7 Step 1447 Action [[-0.11877109]] Reward -8.041935051445124 Loss 0.06110575795173645\n",
      "Episode 7 Step 1448 Action [[-0.18164025]] Reward -7.175999445861617 Loss 0.027453474700450897\n",
      "Episode 7 Step 1449 Action [[0.24168741]] Reward -6.435562698020352 Loss 0.11516666412353516\n",
      "Episode 7 Step 1450 Action [[0.37118585]] Reward -5.861596366910953 Loss 0.08822761476039886\n",
      "Episode 7 Step 1451 Action [[0.60503147]] Reward -5.4427473167612765 Loss 0.037099383771419525\n",
      "Episode 7 Step 1452 Action [[0.84854014]] Reward -5.181808694088108 Loss 0.03917928785085678\n",
      "Episode 7 Step 1453 Action [[0.82943093]] Reward -5.068710750547445 Loss 0.06404492259025574\n",
      "Episode 7 Step 1454 Action [[0.89471204]] Reward -5.10013701923775 Loss 0.06474427878856659\n",
      "Episode 7 Step 1455 Action [[1.00271733]] Reward -5.273821149192972 Loss 0.06830522418022156\n",
      "Episode 7 Step 1456 Action [[1.37416076]] Reward -5.579937525943481 Loss 0.038623157888650894\n",
      "Episode 7 Step 1457 Action [[1.48792042]] Reward -5.9821791759390095 Loss 0.027455560863018036\n",
      "Episode 7 Step 1458 Action [[1.29899657]] Reward -6.466437994850331 Loss 0.12965287268161774\n",
      "Episode 7 Step 1459 Action [[1.29066271]] Reward -7.040144789775493 Loss 0.08518046140670776\n",
      "Episode 7 Step 1460 Action [[1.51235309]] Reward -7.676198947241155 Loss 0.03721517696976662\n",
      "Episode 7 Step 1461 Action [[1.4731276]] Reward -8.324016476600764 Loss 0.6768543720245361\n",
      "Episode 7 Step 1462 Action [[1.32085069]] Reward -8.974884272738445 Loss 0.07746557891368866\n",
      "Episode 7 Step 1463 Action [[1.0116171]] Reward -9.616808242771048 Loss 0.05881822109222412\n",
      "Episode 7 Step 1464 Action [[1.19619159]] Reward -10.249840935344665 Loss 0.4144195020198822\n",
      "Episode 7 Step 1465 Action [[1.07640656]] Reward -9.729097676943478 Loss 0.03632437810301781\n",
      "Episode 7 Step 1466 Action [[0.8192708]] Reward -9.117630479991039 Loss 0.048471905291080475\n",
      "Episode 7 Step 1467 Action [[0.81532118]] Reward -8.596955883404851 Loss 0.0384056456387043\n",
      "Episode 7 Step 1468 Action [[0.88446781]] Reward -8.173730546658424 Loss 0.0295421089977026\n",
      "Episode 7 Step 1469 Action [[1.05171084]] Reward -7.867589509283216 Loss 0.2674601674079895\n",
      "Episode 7 Step 1470 Action [[0.96399384]] Reward -7.699860773977649 Loss 0.08045530319213867\n",
      "Episode 7 Step 1471 Action [[1.0578451]] Reward -7.682521481260071 Loss 0.024506956338882446\n",
      "Episode 7 Step 1472 Action [[0.99913766]] Reward -7.820262063894828 Loss 0.031210720539093018\n",
      "Episode 7 Step 1473 Action [[0.94843231]] Reward -8.106562568947481 Loss 0.027132488787174225\n",
      "Episode 7 Step 1474 Action [[0.69057421]] Reward -8.52631607303433 Loss 0.02095569670200348\n",
      "Episode 7 Step 1475 Action [[1.10082756]] Reward -9.04144814387538 Loss 0.019778672605752945\n",
      "Episode 7 Step 1476 Action [[1.03815399]] Reward -9.675280460802465 Loss 0.017200449481606483\n",
      "Episode 7 Step 1477 Action [[0.87712267]] Reward -10.163196528236892 Loss 0.01570560224354267\n",
      "Episode 7 Step 1478 Action [[0.7061522]] Reward -9.560475001939176 Loss 0.011057957075536251\n",
      "Episode 7 Step 1479 Action [[0.80451739]] Reward -8.933587784214712 Loss 0.01333053782582283\n",
      "Episode 7 Step 1480 Action [[0.86041276]] Reward -8.316490808417724 Loss 0.028675459325313568\n",
      "Episode 7 Step 1481 Action [[0.86052569]] Reward -7.726509452513011 Loss 0.017251795157790184\n",
      "Episode 7 Step 1482 Action [[0.92620706]] Reward -7.182366577153064 Loss 0.015747789293527603\n",
      "Episode 7 Step 1483 Action [[0.69923581]] Reward -6.704491251719856 Loss 0.008896076120436192\n",
      "Episode 7 Step 1484 Action [[0.69541853]] Reward -6.307481001854854 Loss 0.452176570892334\n",
      "Episode 7 Step 1485 Action [[0.60295712]] Reward -6.017795118180008 Loss 0.038123950362205505\n",
      "Episode 7 Step 1486 Action [[1.10089172]] Reward -5.849265924207108 Loss 0.04834583029150963\n",
      "Episode 7 Step 1487 Action [[0.81002648]] Reward -5.793160926037924 Loss 0.12090577185153961\n",
      "Episode 7 Step 1488 Action [[0.75486091]] Reward -5.85387601580782 Loss 0.019385598599910736\n",
      "Episode 7 Step 1489 Action [[0.91088606]] Reward -6.038035529328518 Loss 0.024025127291679382\n",
      "Episode 7 Step 1490 Action [[0.65262825]] Reward -6.329901438212772 Loss 0.012936590239405632\n",
      "Episode 7 Step 1491 Action [[0.47574639]] Reward -6.742794768921108 Loss 0.027232863008975983\n",
      "Episode 7 Step 1492 Action [[0.2551847]] Reward -7.273959674266369 Loss 0.024435168132185936\n",
      "Episode 7 Step 1493 Action [[0.48195574]] Reward -7.9212273872486 Loss 0.005640275776386261\n",
      "Episode 7 Step 1494 Action [[0.41727872]] Reward -8.624925465889413 Loss 0.007375810761004686\n",
      "Episode 7 Step 1495 Action [[0.54336695]] Reward -9.37472501979605 Loss 0.011390313506126404\n",
      "Episode 7 Step 1496 Action [[0.48852588]] Reward -10.11952275969555 Loss 0.0279359370470047\n",
      "Episode 7 Step 1497 Action [[0.5697343]] Reward -10.010898038730549 Loss 0.019577622413635254\n",
      "Episode 7 Step 1498 Action [[0.6317775]] Reward -9.27993985509009 Loss 0.00813923217356205\n",
      "Episode 7 Step 1499 Action [[0.85869172]] Reward -8.604978884173182 Loss 0.09910764545202255\n",
      "Episode 7 Step 1500 Action [[0.99357861]] Reward -8.016072855654425 Loss 0.022998686879873276\n",
      "Episode 7 Step 1501 Action [[0.93951209]] Reward -7.554315169741773 Loss 0.01807549223303795\n",
      "Episode 7 Step 1502 Action [[0.97115093]] Reward -7.248355256538749 Loss 0.009685888886451721\n",
      "Episode 7 Step 1503 Action [[1.07648863]] Reward -7.110895122186961 Loss 0.006991703994572163\n",
      "Episode 7 Step 1504 Action [[1.09929594]] Reward -7.155306609152689 Loss 0.010556165128946304\n",
      "Episode 7 Step 1505 Action [[1.22609169]] Reward -7.38650737087281 Loss 0.009343540295958519\n",
      "Episode 7 Step 1506 Action [[1.12877929]] Reward -7.806515830929763 Loss 0.011349903419613838\n",
      "Episode 7 Step 1507 Action [[1.04694998]] Reward -8.390592440892597 Loss 0.025548430159687996\n",
      "Episode 7 Step 1508 Action [[1.06848376]] Reward -9.108406588020706 Loss 0.006348268128931522\n",
      "Episode 7 Step 1509 Action [[0.91089593]] Reward -9.933655548578956 Loss 0.007514996454119682\n",
      "Episode 7 Step 1510 Action [[1.13326672]] Reward -10.18960743544242 Loss 0.02106061577796936\n",
      "Episode 7 Step 1511 Action [[0.71548079]] Reward -9.459368671403817 Loss 0.01760236732661724\n",
      "Episode 7 Step 1512 Action [[1.01003836]] Reward -8.690100507119851 Loss 0.01012393832206726\n",
      "Episode 7 Step 1513 Action [[0.96290399]] Reward -7.949734063648571 Loss 0.003989687189459801\n",
      "Episode 7 Step 1514 Action [[1.08136417]] Reward -7.2419706116895135 Loss 0.11021348834037781\n",
      "Episode 7 Step 1515 Action [[1.13658619]] Reward -6.599901218901622 Loss 0.009060084819793701\n",
      "Episode 7 Step 1516 Action [[1.08952324]] Reward -6.039020438559155 Loss 0.5025570392608643\n",
      "Episode 7 Step 1517 Action [[1.20766596]] Reward -5.574905705915094 Loss 0.04249045252799988\n",
      "Episode 7 Step 1518 Action [[1.35039396]] Reward -5.225025848129984 Loss 0.03324759006500244\n",
      "Episode 7 Step 1519 Action [[1.27899454]] Reward -4.990863590478186 Loss 0.42783161997795105\n",
      "Episode 7 Step 1520 Action [[1.18225575]] Reward -4.8747441276917005 Loss 0.0412026010453701\n",
      "Episode 7 Step 1521 Action [[1.36705468]] Reward -4.886898919209279 Loss 0.05878344923257828\n",
      "Episode 7 Step 1522 Action [[1.37778573]] Reward -5.02033598106876 Loss 0.019602907821536064\n",
      "Episode 7 Step 1523 Action [[1.19012805]] Reward -5.2708459636667415 Loss 0.019336368888616562\n",
      "Episode 7 Step 1524 Action [[1.33169508]] Reward -5.648657810052537 Loss 0.028734488412737846\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 7 Step 1525 Action [[1.46620947]] Reward -6.129333178726893 Loss 0.02855084277689457\n",
      "Episode 7 Step 1526 Action [[1.30024896]] Reward -6.689272306321627 Loss 0.013514895923435688\n",
      "Episode 7 Step 1527 Action [[1.30253047]] Reward -7.33321745204679 Loss 0.02102811634540558\n",
      "Episode 7 Step 1528 Action [[1.43094326]] Reward -8.028751509227947 Loss 0.01650940626859665\n",
      "Episode 7 Step 1529 Action [[1.3681613]] Reward -8.73237260113034 Loss 0.01577303372323513\n",
      "Episode 7 Step 1530 Action [[1.27721975]] Reward -9.430956281639295 Loss 0.024020075798034668\n",
      "Episode 7 Step 1531 Action [[1.33982821]] Reward -10.102643534613597 Loss 0.15169790387153625\n",
      "Episode 7 Step 1532 Action [[1.41321611]] Reward -9.971322971564824 Loss 0.021467458456754684\n",
      "Episode 7 Step 1533 Action [[1.13609035]] Reward -9.290454655355536 Loss 0.017333833500742912\n",
      "Episode 7 Step 1534 Action [[1.07661201]] Reward -8.714459010564608 Loss 0.515855610370636\n",
      "Episode 7 Step 1535 Action [[1.25118427]] Reward -8.250259806767273 Loss 0.056425806134939194\n",
      "Episode 7 Step 1536 Action [[1.13089546]] Reward -7.916740556381063 Loss 0.044368624687194824\n",
      "Episode 7 Step 1537 Action [[1.03142233]] Reward -7.735653716564303 Loss 0.011424039490520954\n",
      "Episode 7 Step 1538 Action [[1.21176119]] Reward -7.708276627796328 Loss 0.05112498253583908\n",
      "Episode 7 Step 1539 Action [[1.12243574]] Reward -7.843480077600361 Loss 0.03009018674492836\n",
      "Episode 7 Step 1540 Action [[1.29106329]] Reward -8.135069797596024 Loss 0.007772838696837425\n",
      "Episode 7 Step 1541 Action [[1.56087906]] Reward -8.585497337969526 Loss 0.025415459647774696\n",
      "Episode 7 Step 1542 Action [[1.33576331]] Reward -9.198046306734641 Loss 0.020574968308210373\n",
      "Episode 7 Step 1543 Action [[1.34493871]] Reward -9.917794672441488 Loss 0.35614851117134094\n",
      "Episode 7 Step 1544 Action [[1.17708211]] Reward -10.023541582846251 Loss 0.02955888956785202\n",
      "Episode 7 Step 1545 Action [[1.10973166]] Reward -9.368787035621171 Loss 0.02148907259106636\n",
      "Episode 7 Step 1546 Action [[1.42387027]] Reward -8.687996753266471 Loss 0.025845352560281754\n",
      "Episode 7 Step 1547 Action [[1.32419769]] Reward -8.016742755879674 Loss 0.007402435410767794\n",
      "Episode 7 Step 1548 Action [[1.28960809]] Reward -7.354797428155116 Loss 0.009405739605426788\n",
      "Episode 7 Step 1549 Action [[1.38497925]] Reward -6.7327673960750705 Loss 0.016562465578317642\n",
      "Episode 7 Step 1550 Action [[1.21228087]] Reward -6.176517390626534 Loss 0.008972663432359695\n",
      "Episode 7 Step 1551 Action [[1.5048611]] Reward -5.698469379276114 Loss 0.00663316436111927\n",
      "Episode 7 Step 1552 Action [[1.39370464]] Reward -5.323537231255768 Loss 0.006478445138782263\n",
      "Episode 7 Step 1553 Action [[1.25753512]] Reward -5.048246715871367 Loss 0.0062678842805325985\n",
      "Episode 7 Step 1554 Action [[1.14580534]] Reward -4.888076584141912 Loss 0.005871248431503773\n",
      "Episode 7 Step 1555 Action [[1.31909341]] Reward -4.856891068009255 Loss 0.012388106435537338\n",
      "Episode 7 Step 1556 Action [[1.60928817]] Reward -4.95158891577022 Loss 0.014777165837585926\n",
      "Episode 7 Step 1557 Action [[1.3876886]] Reward -5.150748213982319 Loss 0.006864908151328564\n",
      "Episode 7 Step 1558 Action [[1.59255597]] Reward -5.46821516130971 Loss 0.012085484340786934\n",
      "Episode 7 Step 1559 Action [[1.33620964]] Reward -5.876033555510556 Loss 0.01116828341037035\n",
      "Episode 7 Step 1560 Action [[1.51710822]] Reward -6.392967139221042 Loss 0.00792472530156374\n",
      "Episode 7 Step 1561 Action [[1.34212594]] Reward -6.974751712825806 Loss 0.00759307574480772\n",
      "Episode 7 Step 1562 Action [[1.34658421]] Reward -7.628318892620003 Loss 0.008846300654113293\n",
      "Episode 7 Step 1563 Action [[1.2295915]] Reward -8.318018288673299 Loss 0.015513925813138485\n",
      "Episode 7 Step 1564 Action [[1.39801574]] Reward -9.030361595569138 Loss 0.010187257081270218\n",
      "Episode 7 Step 1565 Action [[1.51191548]] Reward -9.707763051110156 Loss 0.0033250951673835516\n",
      "Episode 7 Step 1566 Action [[1.87959558]] Reward -10.323053797412971 Loss 0.00691680982708931\n",
      "Episode 7 Step 1567 Action [[1.48781368]] Reward -9.679652492423028 Loss 0.0028650036547333\n",
      "Episode 7 Step 1568 Action [[1.05212404]] Reward -9.084226906142252 Loss 0.005426749587059021\n",
      "Episode 7 Step 1569 Action [[1.17567116]] Reward -8.604597948919455 Loss 0.005429481156170368\n",
      "Episode 7 Step 1570 Action [[1.47449568]] Reward -8.237752615098916 Loss 0.008200577460229397\n",
      "Episode 7 Step 1571 Action [[1.47298679]] Reward -8.012876214834769 Loss 0.009165072813630104\n",
      "Episode 7 Step 1572 Action [[1.64368094]] Reward -7.955125742049756 Loss 0.38617217540740967\n",
      "Episode 7 Step 1573 Action [[1.81386019]] Reward -8.077700809668757 Loss 0.02204512059688568\n",
      "Episode 7 Step 1574 Action [[1.47724391]] Reward -8.390444532349814 Loss 0.5318448543548584\n",
      "Episode 7 Step 1575 Action [[1.65112579]] Reward -8.857359542253063 Loss 0.06154079735279083\n",
      "Episode 7 Step 1576 Action [[1.27448084]] Reward -9.476562001130308 Loss 0.019370365887880325\n",
      "Episode 7 Step 1577 Action [[1.49906042]] Reward -10.179867849031892 Loss 0.05180216208100319\n",
      "Episode 7 Step 1578 Action [[1.60581169]] Reward -9.74889293142452 Loss 0.05359556898474693\n",
      "Episode 7 Step 1579 Action [[1.74744383]] Reward -9.121453979646544 Loss 0.009643686935305595\n",
      "Episode 7 Step 1580 Action [[2.00874175]] Reward -8.466446072902707 Loss 0.028398944064974785\n",
      "Episode 7 Step 1581 Action [[1.91769064]] Reward -7.803099930007044 Loss 0.4149087369441986\n",
      "Episode 7 Step 1582 Action [[1.7179906]] Reward -7.133940204662462 Loss 0.04914066195487976\n",
      "Episode 7 Step 1583 Action [[1.46791123]] Reward -6.482132777933915 Loss 0.012147403322160244\n",
      "Episode 7 Step 1584 Action [[1.33184738]] Reward -5.879308420904075 Loss 0.03598041832447052\n",
      "Episode 7 Step 1585 Action [[1.45769851]] Reward -5.3619789963205085 Loss 0.010339176282286644\n",
      "Episode 7 Step 1586 Action [[1.45815956]] Reward -4.954899807731921 Loss 0.01643233187496662\n",
      "Episode 7 Step 1587 Action [[1.7936459]] Reward -4.662461008168723 Loss 0.018744874745607376\n",
      "Episode 7 Step 1588 Action [[1.41355918]] Reward -4.483029315770524 Loss 0.01441191416233778\n",
      "Episode 7 Step 1589 Action [[1.52975467]] Reward -4.421108250826453 Loss 0.4139317274093628\n",
      "Episode 7 Step 1590 Action [[1.47491609]] Reward -4.483133159984591 Loss 0.01935591921210289\n",
      "Episode 7 Step 1591 Action [[1.36089313]] Reward -4.670543809467568 Loss 0.028628891333937645\n",
      "Episode 7 Step 1592 Action [[1.06454648]] Reward -4.989015456152276 Loss 0.017362527549266815\n",
      "Episode 7 Step 1593 Action [[0.9284884]] Reward -5.457392375277117 Loss 0.23376764357089996\n",
      "Episode 7 Step 1594 Action [[0.85511572]] Reward -6.072722837625635 Loss 0.01123590487986803\n",
      "Episode 7 Step 1595 Action [[0.45207979]] Reward -6.8213548855301696 Loss 0.12129603326320648\n",
      "Episode 7 Step 1596 Action [[0.69273546]] Reward -7.724832138647967 Loss 0.023733224719762802\n",
      "Episode 7 Step 1597 Action [[0.60385051]] Reward -8.690801800179061 Loss 0.015180078335106373\n",
      "Episode 7 Step 1598 Action [[0.67694351]] Reward -9.707362389882558 Loss 0.013812107965350151\n",
      "Episode 7 Step 1599 Action [[1.09788332]] Reward -10.711288408199417 Loss 0.038791414350271225\n",
      "TOTAL REWARD @ 7-th Episode  : Reward -1460.9359930777412\n",
      "Total Step: 1600\n",
      "\n",
      "Episode : 8 Replay Buffer 1600\n",
      "[-0.03850448 -0.99925843  0.55950626]\n",
      "Episode 8 Step 1600 Action [[0.69690409]] Reward -2.621670139763077 Loss 0.025791220366954803\n",
      "Episode 8 Step 1601 Action [[0.97915868]] Reward -2.6053298979317665 Loss 0.011033023707568645\n",
      "Episode 8 Step 1602 Action [[1.11620192]] Reward -2.764371966968657 Loss 0.044427625834941864\n",
      "Episode 8 Step 1603 Action [[1.42065038]] Reward -3.091672611623746 Loss 0.029938366264104843\n",
      "Episode 8 Step 1604 Action [[1.5879798]] Reward -3.570245485008736 Loss 0.024336189031600952\n",
      "Episode 8 Step 1605 Action [[1.41043303]] Reward -4.196332772822853 Loss 0.03270508348941803\n",
      "Episode 8 Step 1606 Action [[1.11599426]] Reward -4.9990538505901085 Loss 0.04040785878896713\n",
      "Episode 8 Step 1607 Action [[1.25694035]] Reward -6.001406372609582 Loss 0.029190847650170326\n",
      "Episode 8 Step 1608 Action [[1.06025304]] Reward -7.13902643782009 Loss 0.03241344168782234\n",
      "Episode 8 Step 1609 Action [[1.26006658]] Reward -8.417081846262091 Loss 0.03830605000257492\n",
      "Episode 8 Step 1610 Action [[1.42277167]] Reward -9.724693397358607 Loss 0.010310990735888481\n",
      "Episode 8 Step 1611 Action [[0.96235564]] Reward -10.989594571475388 Loss 0.03019827976822853\n",
      "Episode 8 Step 1612 Action [[1.07498147]] Reward -11.026778857120526 Loss 0.02461695671081543\n",
      "Episode 8 Step 1613 Action [[0.96144003]] Reward -9.668979521574936 Loss 0.018528159707784653\n",
      "Episode 8 Step 1614 Action [[1.05205323]] Reward -8.403354527215846 Loss 0.01246017124503851\n",
      "Episode 8 Step 1615 Action [[1.49968993]] Reward -7.267304390180364 Loss 0.022178340703248978\n",
      "Episode 8 Step 1616 Action [[1.63498748]] Reward -6.303665790090712 Loss 0.03146155923604965\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 8 Step 1617 Action [[1.70144251]] Reward -5.600522374577192 Loss 0.009782281704246998\n",
      "Episode 8 Step 1618 Action [[1.86241125]] Reward -5.198893414394938 Loss 0.022375833243131638\n",
      "Episode 8 Step 1619 Action [[1.47805815]] Reward -5.122587481406195 Loss 0.023773595690727234\n",
      "Episode 8 Step 1620 Action [[0.83660163]] Reward -5.36581242308787 Loss 0.018586615100502968\n",
      "Episode 8 Step 1621 Action [[0.94409167]] Reward -5.867319140500679 Loss 0.013848882168531418\n",
      "Episode 8 Step 1622 Action [[1.17745302]] Reward -6.626926749605659 Loss 0.022857101634144783\n",
      "Episode 8 Step 1623 Action [[1.13685345]] Reward -7.647477624431263 Loss 0.01659829169511795\n",
      "Episode 8 Step 1624 Action [[1.18247328]] Reward -8.875775930259293 Loss 0.013981574214994907\n",
      "Episode 8 Step 1625 Action [[1.19273148]] Reward -10.266956511775787 Loss 0.010262217372655869\n",
      "Episode 8 Step 1626 Action [[1.10697062]] Reward -11.285073299468966 Loss 0.005044772755354643\n",
      "Episode 8 Step 1627 Action [[1.0206428]] Reward -10.14046366619887 Loss 0.009832421317696571\n",
      "Episode 8 Step 1628 Action [[1.06413659]] Reward -8.95294032432532 Loss 0.01423192210495472\n",
      "Episode 8 Step 1629 Action [[0.6861229]] Reward -7.7800909460799 Loss 0.006211142987012863\n",
      "Episode 8 Step 1630 Action [[0.93482214]] Reward -6.637904887389544 Loss 0.6550018191337585\n",
      "Episode 8 Step 1631 Action [[0.87586552]] Reward -5.634525577948524 Loss 0.039007045328617096\n",
      "Episode 8 Step 1632 Action [[0.57295335]] Reward -4.771561504576845 Loss 0.024864397943019867\n",
      "Episode 8 Step 1633 Action [[0.68051699]] Reward -4.066771369640985 Loss 0.022348899394273758\n",
      "Episode 8 Step 1634 Action [[0.64516777]] Reward -3.5607464186856213 Loss 0.018790993839502335\n",
      "Episode 8 Step 1635 Action [[0.4421045]] Reward -3.247033715883033 Loss 0.01621869020164013\n",
      "Episode 8 Step 1636 Action [[0.31537355]] Reward -3.1316560203959187 Loss 0.9374110102653503\n",
      "Episode 8 Step 1637 Action [[0.73871623]] Reward -3.230105646260746 Loss 0.07353446632623672\n",
      "Episode 8 Step 1638 Action [[0.81145431]] Reward -3.522662936610113 Loss 0.07717563956975937\n",
      "Episode 8 Step 1639 Action [[0.95527625]] Reward -4.0061988998016576 Loss 0.025135885924100876\n",
      "Episode 8 Step 1640 Action [[0.78510462]] Reward -4.670079190911034 Loss 0.05097716301679611\n",
      "Episode 8 Step 1641 Action [[0.60338301]] Reward -5.53631433097325 Loss 0.07762699574232101\n",
      "Episode 8 Step 1642 Action [[0.5507937]] Reward -6.606998747613204 Loss 0.03726574033498764\n",
      "Episode 8 Step 1643 Action [[0.7112195]] Reward -7.847745139127778 Loss 0.016760310158133507\n",
      "Episode 8 Step 1644 Action [[0.96890797]] Reward -9.178691952838726 Loss 0.057619497179985046\n",
      "Episode 8 Step 1645 Action [[1.43807827]] Reward -10.508846543469874 Loss 0.011090783402323723\n",
      "Episode 8 Step 1646 Action [[1.55792801]] Reward -11.492052264853683 Loss 0.012266557663679123\n",
      "Episode 8 Step 1647 Action [[1.52650112]] Reward -10.107375908169653 Loss 0.008812760934233665\n",
      "Episode 8 Step 1648 Action [[1.3386904]] Reward -8.816166122500237 Loss 0.011514106765389442\n",
      "Episode 8 Step 1649 Action [[1.59547481]] Reward -7.68381660902883 Loss 0.010268384590744972\n",
      "Episode 8 Step 1650 Action [[1.5735193]] Reward -6.7287030385276765 Loss 0.03121171146631241\n",
      "Episode 8 Step 1651 Action [[1.39978054]] Reward -6.021518261986982 Loss 0.014255374670028687\n",
      "Episode 8 Step 1652 Action [[1.11513089]] Reward -5.589147922636754 Loss 0.01689249649643898\n",
      "Episode 8 Step 1653 Action [[0.97260119]] Reward -5.4297548903471355 Loss 0.017017733305692673\n",
      "Episode 8 Step 1654 Action [[1.02914119]] Reward -5.530426590185631 Loss 0.012977476231753826\n",
      "Episode 8 Step 1655 Action [[0.81699562]] Reward -5.890779099878385 Loss 0.13803620636463165\n",
      "Episode 8 Step 1656 Action [[0.9176978]] Reward -6.486403026214909 Loss 0.016876425594091415\n",
      "Episode 8 Step 1657 Action [[0.59661464]] Reward -7.312635300773703 Loss 0.009984737262129784\n",
      "Episode 8 Step 1658 Action [[0.75728536]] Reward -8.29796007347644 Loss 0.026532957330346107\n",
      "Episode 8 Step 1659 Action [[0.49721878]] Reward -9.43899662723364 Loss 0.011670898646116257\n",
      "Episode 8 Step 1660 Action [[0.5822335]] Reward -10.63430386430161 Loss 1.0996150970458984\n",
      "Episode 8 Step 1661 Action [[0.86973786]] Reward -10.462364180017628 Loss 0.11175195127725601\n",
      "Episode 8 Step 1662 Action [[1.02862027]] Reward -9.431021648863839 Loss 0.2654725909233093\n",
      "Episode 8 Step 1663 Action [[0.61676145]] Reward -8.40245787181123 Loss 0.16272908449172974\n",
      "Episode 8 Step 1664 Action [[1.03027139]] Reward -7.374896860952813 Loss 0.1874738335609436\n",
      "Episode 8 Step 1665 Action [[0.97023341]] Reward -6.455562046727898 Loss 0.08430160582065582\n",
      "Episode 8 Step 1666 Action [[1.11234129]] Reward -5.633856251398672 Loss 0.03922918811440468\n",
      "Episode 8 Step 1667 Action [[0.88989709]] Reward -4.946813451376136 Loss 0.20913460850715637\n",
      "Episode 8 Step 1668 Action [[0.94015903]] Reward -4.396472183114186 Loss 0.05619232356548309\n",
      "Episode 8 Step 1669 Action [[0.78530091]] Reward -4.010810166692263 Loss 0.029154185205698013\n",
      "Episode 8 Step 1670 Action [[0.90645322]] Reward -3.794448373296378 Loss 0.09628758579492569\n",
      "Episode 8 Step 1671 Action [[0.89111019]] Reward -3.7539237634192197 Loss 0.11034552752971649\n",
      "Episode 8 Step 1672 Action [[0.98842168]] Reward -3.887960355256329 Loss 0.02010287344455719\n",
      "Episode 8 Step 1673 Action [[0.95039901]] Reward -4.191695380258957 Loss 0.07531186938285828\n",
      "Episode 8 Step 1674 Action [[0.79828141]] Reward -4.668152863173969 Loss 0.09137673676013947\n",
      "Episode 8 Step 1675 Action [[0.96143857]] Reward -5.32856345370875 Loss 0.03032255731523037\n",
      "Episode 8 Step 1676 Action [[0.78170569]] Reward -6.1377011246382 Loss 0.09178591519594193\n",
      "Episode 8 Step 1677 Action [[0.69960053]] Reward -7.106127220132829 Loss 0.06921571493148804\n",
      "Episode 8 Step 1678 Action [[0.93680309]] Reward -8.200398109270989 Loss 0.013657363131642342\n",
      "Episode 8 Step 1679 Action [[1.1610996]] Reward -9.331642448408257 Loss 0.04903301224112511\n",
      "Episode 8 Step 1680 Action [[1.01573892]] Reward -10.433488720009779 Loss 0.055670592933893204\n",
      "Episode 8 Step 1681 Action [[0.78634964]] Reward -10.805974590304727 Loss 0.00675693666562438\n",
      "Episode 8 Step 1682 Action [[0.73331087]] Reward -9.66831435502726 Loss 0.06355303525924683\n",
      "Episode 8 Step 1683 Action [[0.92662494]] Reward -8.5950999977456 Loss 0.02676287665963173\n",
      "Episode 8 Step 1684 Action [[1.03597745]] Reward -7.617300441970462 Loss 0.1265183985233307\n",
      "Episode 8 Step 1685 Action [[1.00846151]] Reward -6.794722721630818 Loss 0.024703986942768097\n",
      "Episode 8 Step 1686 Action [[1.18315784]] Reward -6.174398494057917 Loss 0.07824616879224777\n",
      "Episode 8 Step 1687 Action [[1.07802868]] Reward -5.780008214142629 Loss 0.143891379237175\n",
      "Episode 8 Step 1688 Action [[0.48491696]] Reward -5.636730128573207 Loss 0.020801929756999016\n",
      "Episode 8 Step 1689 Action [[0.47929139]] Reward -5.721093736140387 Loss 0.01565726101398468\n",
      "Episode 8 Step 1690 Action [[0.65995914]] Reward -6.016458447474241 Loss 0.016157912090420723\n",
      "Episode 8 Step 1691 Action [[0.51663094]] Reward -6.5305572520288235 Loss 0.01634487509727478\n",
      "Episode 8 Step 1692 Action [[0.22059869]] Reward -7.232574985093524 Loss 0.02360759675502777\n",
      "Episode 8 Step 1693 Action [[0.24048773]] Reward -8.0678362839256 Loss 0.14287729561328888\n",
      "Episode 8 Step 1694 Action [[0.60084722]] Reward -9.018432900169804 Loss 0.006063244305551052\n",
      "Episode 8 Step 1695 Action [[1.02293945]] Reward -10.090404611710989 Loss 0.02641882747411728\n",
      "Episode 8 Step 1696 Action [[0.84147387]] Reward -10.655425011543636 Loss 0.02311382256448269\n",
      "Episode 8 Step 1697 Action [[0.8608793]] Reward -9.69563149096517 Loss 0.009393110871315002\n",
      "Episode 8 Step 1698 Action [[0.60592489]] Reward -8.719630563559422 Loss 0.02170480787754059\n",
      "Episode 8 Step 1699 Action [[0.92534177]] Reward -7.748019636882847 Loss 0.009825292974710464\n",
      "Episode 8 Step 1700 Action [[0.99999557]] Reward -6.86035195754212 Loss 0.008480981923639774\n",
      "Episode 8 Step 1701 Action [[0.98178549]] Reward -6.062339114191908 Loss 0.015131926164031029\n",
      "Episode 8 Step 1702 Action [[0.85014143]] Reward -5.374736779896322 Loss 0.01036955788731575\n",
      "Episode 8 Step 1703 Action [[0.91031566]] Reward -4.818318616101434 Loss 0.254506379365921\n",
      "Episode 8 Step 1704 Action [[0.77497275]] Reward -4.417147494307881 Loss 0.03434981778264046\n",
      "Episode 8 Step 1705 Action [[1.02834463]] Reward -4.177902810792872 Loss 0.7563884258270264\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 8 Step 1706 Action [[1.09914366]] Reward -4.104487656833953 Loss 0.05304613709449768\n",
      "Episode 8 Step 1707 Action [[1.31268039]] Reward -4.187806668578294 Loss 0.011411393992602825\n",
      "Episode 8 Step 1708 Action [[1.33144356]] Reward -4.414318261482858 Loss 0.024355458095669746\n",
      "Episode 8 Step 1709 Action [[1.4540046]] Reward -4.781338983810248 Loss 0.03073550947010517\n",
      "Episode 8 Step 1710 Action [[1.20004497]] Reward -5.273047099753735 Loss 0.018577376380562782\n",
      "Episode 8 Step 1711 Action [[1.23891621]] Reward -5.909815982058516 Loss 0.1601080745458603\n",
      "Episode 8 Step 1712 Action [[1.16110518]] Reward -6.660524796771847 Loss 0.031359605491161346\n",
      "Episode 8 Step 1713 Action [[1.33630311]] Reward -7.513023469607509 Loss 0.022574327886104584\n",
      "Episode 8 Step 1714 Action [[1.33767897]] Reward -8.405664604768836 Loss 0.030654842033982277\n",
      "Episode 8 Step 1715 Action [[1.38330877]] Reward -9.313686226347524 Loss 0.03368628770112991\n",
      "Episode 8 Step 1716 Action [[1.56276672]] Reward -10.191502512151516 Loss 0.01336190290749073\n",
      "Episode 8 Step 1717 Action [[1.41193131]] Reward -10.380385907234986 Loss 0.9258418083190918\n",
      "Episode 8 Step 1718 Action [[1.14191412]] Reward -9.459780509373646 Loss 0.28189101815223694\n",
      "Episode 8 Step 1719 Action [[1.04661402]] Reward -8.645867017875052 Loss 0.03756711632013321\n",
      "Episode 8 Step 1720 Action [[1.33775304]] Reward -7.952842960192868 Loss 0.03403565287590027\n",
      "Episode 8 Step 1721 Action [[1.2366157]] Reward -7.401172356359652 Loss 0.05912994220852852\n",
      "Episode 8 Step 1722 Action [[1.4180402]] Reward -7.037006845549112 Loss 0.1498037874698639\n",
      "Episode 8 Step 1723 Action [[1.44896639]] Reward -6.876015084644222 Loss 0.6764865517616272\n",
      "Episode 8 Step 1724 Action [[1.34860347]] Reward -6.937022712212664 Loss 0.17394444346427917\n",
      "Episode 8 Step 1725 Action [[1.57034595]] Reward -7.21635464026994 Loss 0.19730111956596375\n",
      "Episode 8 Step 1726 Action [[1.07555982]] Reward -7.719961373073712 Loss 0.046188466250896454\n",
      "Episode 8 Step 1727 Action [[1.02825177]] Reward -8.381746301865016 Loss 0.10299117863178253\n",
      "Episode 8 Step 1728 Action [[1.25840342]] Reward -9.181224075324504 Loss 1.0543403625488281\n",
      "Episode 8 Step 1729 Action [[1.20419628]] Reward -10.112220529457565 Loss 0.7068746089935303\n",
      "Episode 8 Step 1730 Action [[0.69821948]] Reward -10.197206759315234 Loss 0.37388622760772705\n",
      "Episode 8 Step 1731 Action [[0.53386057]] Reward -9.37324255342757 Loss 0.1466384083032608\n",
      "Episode 8 Step 1732 Action [[0.62209582]] Reward -8.539205999043869 Loss 0.05177135765552521\n",
      "Episode 8 Step 1733 Action [[0.68288709]] Reward -7.743188120050924 Loss 1.2189643383026123\n",
      "Episode 8 Step 1734 Action [[0.62315836]] Reward -7.0103990657907636 Loss 1.3448400497436523\n",
      "Episode 8 Step 1735 Action [[0.79637748]] Reward -6.363131725509523 Loss 0.5282687544822693\n",
      "Episode 8 Step 1736 Action [[0.92663849]] Reward -5.83220402028283 Loss 0.42999106645584106\n",
      "Episode 8 Step 1737 Action [[1.34302815]] Reward -5.424839310302529 Loss 0.06659868359565735\n",
      "Episode 8 Step 1738 Action [[1.63331882]] Reward -5.142056750898469 Loss 0.17516958713531494\n",
      "Episode 8 Step 1739 Action [[1.48533917]] Reward -4.965476243041644 Loss 0.42707327008247375\n",
      "Episode 8 Step 1740 Action [[1.41176768]] Reward -4.894681586498576 Loss 0.12381882965564728\n",
      "Episode 8 Step 1741 Action [[1.00020658]] Reward -4.938066319186074 Loss 0.09621647000312805\n",
      "Episode 8 Step 1742 Action [[1.13594716]] Reward -5.120570211682355 Loss 0.1034945696592331\n",
      "Episode 8 Step 1743 Action [[0.84920583]] Reward -5.43114615978713 Loss 0.3429781198501587\n",
      "Episode 8 Step 1744 Action [[0.73195668]] Reward -5.888270839285628 Loss 0.09515365213155746\n",
      "Episode 8 Step 1745 Action [[0.66726921]] Reward -6.485635420482835 Loss 0.20274092257022858\n",
      "Episode 8 Step 1746 Action [[0.65151407]] Reward -7.207922648329223 Loss 0.11145232617855072\n",
      "Episode 8 Step 1747 Action [[0.87493273]] Reward -8.0281325291806 Loss 0.06838192045688629\n",
      "Episode 8 Step 1748 Action [[1.01085947]] Reward -8.883457425299003 Loss 0.15220646560192108\n",
      "Episode 8 Step 1749 Action [[0.97680701]] Reward -9.734440161975185 Loss 0.0843149870634079\n",
      "Episode 8 Step 1750 Action [[0.79660537]] Reward -10.559564766168059 Loss 0.09021101891994476\n",
      "Episode 8 Step 1751 Action [[1.19519301]] Reward -9.827563184547001 Loss 0.06089906767010689\n",
      "Episode 8 Step 1752 Action [[1.15820806]] Reward -8.987336114022387 Loss 0.024748846888542175\n",
      "Episode 8 Step 1753 Action [[1.3895401]] Reward -8.254888664172537 Loss 0.11808528751134872\n",
      "Episode 8 Step 1754 Action [[1.35491951]] Reward -7.656734763882472 Loss 0.22274179756641388\n",
      "Episode 8 Step 1755 Action [[1.34300773]] Reward -7.237597250794121 Loss 0.09132474660873413\n",
      "Episode 8 Step 1756 Action [[1.08410421]] Reward -7.0143641873219895 Loss 0.03723149001598358\n",
      "Episode 8 Step 1757 Action [[0.83950761]] Reward -6.9885642148273535 Loss 1.3333121538162231\n",
      "Episode 8 Step 1758 Action [[0.9465474]] Reward -7.142610401149764 Loss 0.044996581971645355\n",
      "Episode 8 Step 1759 Action [[1.60499329]] Reward -7.477137621161448 Loss 0.197360098361969\n",
      "Episode 8 Step 1760 Action [[1.65015046]] Reward -8.034224787100621 Loss 0.22039388120174408\n",
      "Episode 8 Step 1761 Action [[1.43876795]] Reward -8.78889790280842 Loss 0.06024804338812828\n",
      "Episode 8 Step 1762 Action [[1.14149213]] Reward -9.683132345225975 Loss 0.40401020646095276\n",
      "Episode 8 Step 1763 Action [[1.1751016]] Reward -10.542696337183438 Loss 0.09265801310539246\n",
      "Episode 8 Step 1764 Action [[1.02906225]] Reward -9.760460065996757 Loss 0.06925617903470993\n",
      "Episode 8 Step 1765 Action [[0.52455476]] Reward -8.936347174024764 Loss 1.0211195945739746\n",
      "Episode 8 Step 1766 Action [[0.52339453]] Reward -8.08979720704434 Loss 0.19658176600933075\n",
      "Episode 8 Step 1767 Action [[0.8256815]] Reward -7.297062774430627 Loss 0.04370465502142906\n",
      "Episode 8 Step 1768 Action [[0.64403217]] Reward -6.597410523780423 Loss 0.1309126913547516\n",
      "Episode 8 Step 1769 Action [[0.64940966]] Reward -5.989298994696566 Loss 0.3350299596786499\n",
      "Episode 8 Step 1770 Action [[0.93297416]] Reward -5.508390301455651 Loss 0.060997359454631805\n",
      "Episode 8 Step 1771 Action [[1.30101011]] Reward -5.169010469586499 Loss 0.18208423256874084\n",
      "Episode 8 Step 1772 Action [[1.47341615]] Reward -4.959317935795261 Loss 0.1539749801158905\n",
      "Episode 8 Step 1773 Action [[0.85229967]] Reward -4.863477263602709 Loss 0.04116413742303848\n",
      "Episode 8 Step 1774 Action [[1.09927016]] Reward -4.906614374204883 Loss 0.14302153885364532\n",
      "Episode 8 Step 1775 Action [[1.40218389]] Reward -5.087513043772994 Loss 0.08992033451795578\n",
      "Episode 8 Step 1776 Action [[1.23608439]] Reward -5.380378402023674 Loss 0.13261979818344116\n",
      "Episode 8 Step 1777 Action [[1.41420898]] Reward -5.796059818129063 Loss 0.12411428987979889\n",
      "Episode 8 Step 1778 Action [[1.37440178]] Reward -6.303690396944767 Loss 0.10929560661315918\n",
      "Episode 8 Step 1779 Action [[1.26380298]] Reward -6.897617529549233 Loss 0.05589878186583519\n",
      "Episode 8 Step 1780 Action [[1.09945509]] Reward -7.567251178551733 Loss 0.041408732533454895\n",
      "Episode 8 Step 1781 Action [[1.21526839]] Reward -8.3004105892742 Loss 0.11299580335617065\n",
      "Episode 8 Step 1782 Action [[1.35336185]] Reward -9.042937792727027 Loss 0.028355272486805916\n",
      "Episode 8 Step 1783 Action [[1.01864922]] Reward -9.753723508470225 Loss 0.0758475586771965\n",
      "Episode 8 Step 1784 Action [[0.55635904]] Reward -10.415692565772227 Loss 0.0652613639831543\n",
      "Episode 8 Step 1785 Action [[0.85077105]] Reward -9.669947827466427 Loss 0.04210497438907623\n",
      "Episode 8 Step 1786 Action [[0.74888684]] Reward -8.958598465585911 Loss 0.04770984500646591\n",
      "Episode 8 Step 1787 Action [[0.69922606]] Reward -8.333946806864923 Loss 0.030635051429271698\n",
      "Episode 8 Step 1788 Action [[0.68512036]] Reward -7.815675620162509 Loss 0.015483367256820202\n",
      "Episode 8 Step 1789 Action [[0.47351521]] Reward -7.422841999979031 Loss 0.016985008493065834\n",
      "Episode 8 Step 1790 Action [[0.69630415]] Reward -7.168692205815814 Loss 0.9404147863388062\n",
      "Episode 8 Step 1791 Action [[0.53361696]] Reward -7.06175561620829 Loss 0.06615480035543442\n",
      "Episode 8 Step 1792 Action [[0.49609436]] Reward -7.1086107597179 Loss 0.03126204386353493\n",
      "Episode 8 Step 1793 Action [[0.59272296]] Reward -7.303595206598723 Loss 0.034616366028785706\n",
      "Episode 8 Step 1794 Action [[0.53203867]] Reward -7.64684828107476 Loss 0.04798096418380737\n",
      "Episode 8 Step 1795 Action [[0.68070218]] Reward -8.120587255510207 Loss 0.017080385237932205\n",
      "Episode 8 Step 1796 Action [[0.50782908]] Reward -8.72055068094622 Loss 0.022806528955698013\n",
      "Episode 8 Step 1797 Action [[0.5386212]] Reward -9.39949659435964 Loss 0.015534671023488045\n",
      "Episode 8 Step 1798 Action [[0.72559486]] Reward -10.138801324212663 Loss 0.058642178773880005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 8 Step 1799 Action [[0.58148688]] Reward -9.885036536125575 Loss 0.019475150853395462\n",
      "TOTAL REWARD @ 8-th Episode  : Reward -1424.199901968875\n",
      "Total Step: 1800\n",
      "\n",
      "Episode : 9 Replay Buffer 1800\n",
      "[-0.62331326 -0.78197224  0.53905689]\n",
      "Episode 9 Step 1800 Action [[0.92064946]] Reward -5.064405007987237 Loss 1.225677251815796\n",
      "Episode 9 Step 1801 Action [[0.77083393]] Reward -5.015590713073616 Loss 0.1051422655582428\n",
      "Episode 9 Step 1802 Action [[0.95183691]] Reward -5.115664696240842 Loss 0.8778871297836304\n",
      "Episode 9 Step 1803 Action [[0.5851745]] Reward -5.354244539869792 Loss 0.07905513048171997\n",
      "Episode 9 Step 1804 Action [[0.48706317]] Reward -5.754702893003505 Loss 0.01647716388106346\n",
      "Episode 9 Step 1805 Action [[0.48924101]] Reward -6.312154095467243 Loss 0.08872745931148529\n",
      "Episode 9 Step 1806 Action [[0.47759288]] Reward -7.007678246512509 Loss 0.23649629950523376\n",
      "Episode 9 Step 1807 Action [[0.65423427]] Reward -7.818707321392543 Loss 0.990077793598175\n",
      "Episode 9 Step 1808 Action [[0.76783078]] Reward -8.690120735323289 Loss 0.19149038195610046\n",
      "Episode 9 Step 1809 Action [[0.75716933]] Reward -9.580962117246319 Loss 0.14218789339065552\n",
      "Episode 9 Step 1810 Action [[0.35216648]] Reward -10.461982565233782 Loss 0.1049165353178978\n",
      "Episode 9 Step 1811 Action [[0.33558597]] Reward -10.026643045270756 Loss 0.12998579442501068\n",
      "Episode 9 Step 1812 Action [[0.67381807]] Reward -9.150027089469667 Loss 0.19304800033569336\n",
      "Episode 9 Step 1813 Action [[0.76120843]] Reward -8.315279862360768 Loss 0.25999993085861206\n",
      "Episode 9 Step 1814 Action [[0.43350568]] Reward -7.584565356941218 Loss 0.23862850666046143\n",
      "Episode 9 Step 1815 Action [[0.41571838]] Reward -7.000856947567065 Loss 0.3314059376716614\n",
      "Episode 9 Step 1816 Action [[0.71049984]] Reward -6.565020040379901 Loss 0.17900419235229492\n",
      "Episode 9 Step 1817 Action [[0.65324769]] Reward -6.298468850437626 Loss 0.07605575025081635\n",
      "Episode 9 Step 1818 Action [[0.84250588]] Reward -6.226954981098095 Loss 0.28000056743621826\n",
      "Episode 9 Step 1819 Action [[0.81254606]] Reward -6.362487383182136 Loss 0.14633852243423462\n",
      "Episode 9 Step 1820 Action [[1.16797416]] Reward -6.70595549859362 Loss 0.08063649386167526\n",
      "Episode 9 Step 1821 Action [[0.95398368]] Reward -7.276264468570184 Loss 0.205356627702713\n",
      "Episode 9 Step 1822 Action [[1.00630316]] Reward -8.02810505464219 Loss 0.1793796420097351\n",
      "Episode 9 Step 1823 Action [[1.27124572]] Reward -8.944127824752774 Loss 0.021791867911815643\n",
      "Episode 9 Step 1824 Action [[1.53824365]] Reward -10.016361591279392 Loss 0.19106440246105194\n",
      "Episode 9 Step 1825 Action [[1.36183365]] Reward -10.624461351052116 Loss 0.17826411128044128\n",
      "Episode 9 Step 1826 Action [[0.90441489]] Reward -9.711189187725473 Loss 0.06271088868379593\n",
      "Episode 9 Step 1827 Action [[0.67217081]] Reward -8.733858477716453 Loss 0.1646808683872223\n",
      "Episode 9 Step 1828 Action [[0.54779901]] Reward -7.759080040466389 Loss 0.07480151951313019\n",
      "Episode 9 Step 1829 Action [[0.62650161]] Reward -6.843150418198344 Loss 0.026911722496151924\n",
      "Episode 9 Step 1830 Action [[0.67581811]] Reward -6.037816096192898 Loss 0.0728059858083725\n",
      "Episode 9 Step 1831 Action [[0.21586776]] Reward -5.365657034325112 Loss 0.09059654921293259\n",
      "Episode 9 Step 1832 Action [[0.43174507]] Reward -4.838118229216965 Loss 1.3447308540344238\n",
      "Episode 9 Step 1833 Action [[0.55728989]] Reward -4.502999850280051 Loss 0.2174002230167389\n",
      "Episode 9 Step 1834 Action [[0.73652072]] Reward -4.352468660316001 Loss 0.39270660281181335\n",
      "Episode 9 Step 1835 Action [[0.54642344]] Reward -4.377775445861851 Loss 0.08796913176774979\n",
      "Episode 9 Step 1836 Action [[0.11497076]] Reward -4.586155870817707 Loss 0.1424836814403534\n",
      "Episode 9 Step 1837 Action [[0.07732007]] Reward -5.010740052217694 Loss 0.17252838611602783\n",
      "Episode 9 Step 1838 Action [[0.28949516]] Reward -5.648252498499571 Loss 0.10511418431997299\n",
      "Episode 9 Step 1839 Action [[0.31200836]] Reward -6.461627734481197 Loss 0.18155837059020996\n",
      "Episode 9 Step 1840 Action [[0.26206411]] Reward -7.433331878413787 Loss 1.1985613107681274\n",
      "Episode 9 Step 1841 Action [[0.46325124]] Reward -8.535810874899498 Loss 0.2634413242340088\n",
      "Episode 9 Step 1842 Action [[0.52403475]] Reward -9.686996847350303 Loss 0.06060042604804039\n",
      "Episode 9 Step 1843 Action [[0.33509592]] Reward -10.841494452841854 Loss 0.14912229776382446\n",
      "Episode 9 Step 1844 Action [[0.20788674]] Reward -10.399954637358071 Loss 0.5215723514556885\n",
      "Episode 9 Step 1845 Action [[0.40729661]] Reward -9.289482366311763 Loss 0.12133228033781052\n",
      "Episode 9 Step 1846 Action [[0.44908169]] Reward -8.21506776041335 Loss 0.12350189685821533\n",
      "Episode 9 Step 1847 Action [[0.67343662]] Reward -7.242244389666988 Loss 0.13398651778697968\n",
      "Episode 9 Step 1848 Action [[0.52304275]] Reward -6.408958473681367 Loss 0.094950370490551\n",
      "Episode 9 Step 1849 Action [[0.31417611]] Reward -5.773848565665557 Loss 0.10104166716337204\n",
      "Episode 9 Step 1850 Action [[0.02601111]] Reward -5.347870723942684 Loss 1.248021125793457\n",
      "Episode 9 Step 1851 Action [[0.09531152]] Reward -5.126433592676687 Loss 0.06566886603832245\n",
      "Episode 9 Step 1852 Action [[0.19488272]] Reward -5.105206817225862 Loss 0.14300456643104553\n",
      "Episode 9 Step 1853 Action [[0.31503301]] Reward -5.294961033204772 Loss 0.13073891401290894\n",
      "Episode 9 Step 1854 Action [[0.47310072]] Reward -5.704312572576298 Loss 0.06206946074962616\n",
      "Episode 9 Step 1855 Action [[0.73131826]] Reward -6.339605240567746 Loss 0.9603638052940369\n",
      "Episode 9 Step 1856 Action [[0.697669]] Reward -7.208647019965779 Loss 0.7069454789161682\n",
      "Episode 9 Step 1857 Action [[0.6739662]] Reward -8.267693016681669 Loss 0.38085609674453735\n",
      "Episode 9 Step 1858 Action [[0.83645522]] Reward -9.47140965549729 Loss 0.406817764043808\n",
      "Episode 9 Step 1859 Action [[0.55256395]] Reward -10.790356739822515 Loss 0.11350840330123901\n",
      "Episode 9 Step 1860 Action [[0.33140909]] Reward -10.585280543431676 Loss 0.3083651065826416\n",
      "Episode 9 Step 1861 Action [[-0.02793409]] Reward -9.448345772621815 Loss 0.19911906123161316\n",
      "Episode 9 Step 1862 Action [[0.02866045]] Reward -8.304918469598562 Loss 1.4263323545455933\n",
      "Episode 9 Step 1863 Action [[0.30925098]] Reward -7.24727045317727 Loss 1.2498100996017456\n",
      "Episode 9 Step 1864 Action [[0.29356121]] Reward -6.326538862288337 Loss 1.565703272819519\n",
      "Episode 9 Step 1865 Action [[0.59075242]] Reward -5.548330391144972 Loss 0.13919061422348022\n",
      "Episode 9 Step 1866 Action [[0.61346015]] Reward -4.9460652076275 Loss 0.42773565649986267\n",
      "Episode 9 Step 1867 Action [[0.99138129]] Reward -4.512422060547035 Loss 0.6328007578849792\n",
      "Episode 9 Step 1868 Action [[1.24847344]] Reward -4.251646179721379 Loss 0.335049569606781\n",
      "Episode 9 Step 1869 Action [[1.20453393]] Reward -4.1421623174671875 Loss 0.4451274871826172\n",
      "Episode 9 Step 1870 Action [[1.13441234]] Reward -4.179057975084385 Loss 0.4514123499393463\n",
      "Episode 9 Step 1871 Action [[1.31485406]] Reward -4.3687247249746255 Loss 0.11061246693134308\n",
      "Episode 9 Step 1872 Action [[1.18709477]] Reward -4.697725823703897 Loss 0.37207746505737305\n",
      "Episode 9 Step 1873 Action [[1.20162271]] Reward -5.175245702511697 Loss 0.7256811857223511\n",
      "Episode 9 Step 1874 Action [[1.17798571]] Reward -5.78877576994515 Loss 0.07441481202840805\n",
      "Episode 9 Step 1875 Action [[1.15485611]] Reward -6.526708410265979 Loss 0.46191298961639404\n",
      "Episode 9 Step 1876 Action [[1.56597519]] Reward -7.368726558106448 Loss 0.7567482590675354\n",
      "Episode 9 Step 1877 Action [[1.49691389]] Reward -8.226418629036342 Loss 0.1678338497877121\n",
      "Episode 9 Step 1878 Action [[1.16814643]] Reward -9.10352055179133 Loss 0.46385958790779114\n",
      "Episode 9 Step 1879 Action [[1.28295064]] Reward -10.002285498345127 Loss 0.8875123262405396\n",
      "Episode 9 Step 1880 Action [[1.29945666]] Reward -10.583784767037924 Loss 1.1456209421157837\n",
      "Episode 9 Step 1881 Action [[0.91080992]] Reward -9.644711782256595 Loss 0.9526629447937012\n",
      "Episode 9 Step 1882 Action [[0.94185849]] Reward -8.80295174601354 Loss 1.3986237049102783\n",
      "Episode 9 Step 1883 Action [[1.18969254]] Reward -8.059172709243025 Loss 0.9218855500221252\n",
      "Episode 9 Step 1884 Action [[1.43733914]] Reward -7.445708007631977 Loss 0.21196091175079346\n",
      "Episode 9 Step 1885 Action [[1.47634725]] Reward -7.009557178683842 Loss 1.7246644496917725\n",
      "Episode 9 Step 1886 Action [[1.26572864]] Reward -6.787925663532165 Loss 0.2551828622817993\n",
      "Episode 9 Step 1887 Action [[1.39826175]] Reward -6.787460831626834 Loss 0.29767441749572754\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 9 Step 1888 Action [[0.9290725]] Reward -7.0099615827605115 Loss 0.370718777179718\n",
      "Episode 9 Step 1889 Action [[0.99033729]] Reward -7.418171025180518 Loss 1.0942822694778442\n",
      "Episode 9 Step 1890 Action [[0.81509015]] Reward -8.003301087514467 Loss 0.03900531679391861\n",
      "Episode 9 Step 1891 Action [[0.98588983]] Reward -8.723629084804823 Loss 0.16163238883018494\n",
      "Episode 9 Step 1892 Action [[1.29683264]] Reward -9.574322396741538 Loss 0.16476938128471375\n",
      "Episode 9 Step 1893 Action [[1.19719422]] Reward -10.547193086046882 Loss 0.0784834548830986\n",
      "Episode 9 Step 1894 Action [[1.10119075]] Reward -9.850707670792053 Loss 0.114591583609581\n",
      "Episode 9 Step 1895 Action [[1.0432014]] Reward -9.02932544866479 Loss 0.14952826499938965\n",
      "Episode 9 Step 1896 Action [[0.71408378]] Reward -8.197483941860288 Loss 0.05572252720594406\n",
      "Episode 9 Step 1897 Action [[0.92735981]] Reward -7.378227611049719 Loss 0.027209188789129257\n",
      "Episode 9 Step 1898 Action [[0.80427379]] Reward -6.6385707131274945 Loss 0.3352718651294708\n",
      "Episode 9 Step 1899 Action [[0.67645184]] Reward -5.985231258539433 Loss 0.035971127450466156\n",
      "Episode 9 Step 1900 Action [[0.56498891]] Reward -5.449186305612808 Loss 0.0352209210395813\n",
      "Episode 9 Step 1901 Action [[0.49189733]] Reward -5.0567216537471955 Loss 1.5871158838272095\n",
      "Episode 9 Step 1902 Action [[0.66048003]] Reward -4.827460567494699 Loss 0.13138291239738464\n",
      "Episode 9 Step 1903 Action [[0.71940936]] Reward -4.766358934392216 Loss 0.47069692611694336\n",
      "Episode 9 Step 1904 Action [[0.89366666]] Reward -4.868322610923461 Loss 0.1623675674200058\n",
      "Episode 9 Step 1905 Action [[1.16600756]] Reward -5.121653747124397 Loss 0.13695074617862701\n",
      "Episode 9 Step 1906 Action [[0.95878993]] Reward -5.500673026106994 Loss 1.7707772254943848\n",
      "Episode 9 Step 1907 Action [[0.92179803]] Reward -6.019770905043434 Loss 0.031077824532985687\n",
      "Episode 9 Step 1908 Action [[0.74872963]] Reward -6.662763970261535 Loss 0.2075785994529724\n",
      "Episode 9 Step 1909 Action [[0.65389683]] Reward -7.426817520548296 Loss 0.22816038131713867\n",
      "Episode 9 Step 1910 Action [[0.7103752]] Reward -8.28590325957075 Loss 0.477100133895874\n",
      "Episode 9 Step 1911 Action [[0.7413606]] Reward -9.190632274529746 Loss 0.07207643985748291\n",
      "Episode 9 Step 1912 Action [[0.72788671]] Reward -10.101190109160646 Loss 0.07880005985498428\n",
      "Episode 9 Step 1913 Action [[0.70871231]] Reward -10.436464319902623 Loss 0.09753832221031189\n",
      "Episode 9 Step 1914 Action [[0.47395234]] Reward -9.523142653610243 Loss 0.04395437240600586\n",
      "Episode 9 Step 1915 Action [[0.69059762]] Reward -8.675690006586812 Loss 0.0986747145652771\n",
      "Episode 9 Step 1916 Action [[0.93073718]] Reward -7.90223182149191 Loss 0.2081313580274582\n",
      "Episode 9 Step 1917 Action [[0.78613691]] Reward -7.249820168379071 Loss 1.2098428010940552\n",
      "Episode 9 Step 1918 Action [[0.87507511]] Reward -6.766716373833767 Loss 0.030649345368146896\n",
      "Episode 9 Step 1919 Action [[0.85671129]] Reward -6.465204756947182 Loss 0.026911258697509766\n",
      "Episode 9 Step 1920 Action [[0.88329436]] Reward -6.363568194111379 Loss 0.058702364563941956\n",
      "Episode 9 Step 1921 Action [[1.39944555]] Reward -6.469859659111323 Loss 0.04168519005179405\n",
      "Episode 9 Step 1922 Action [[1.61439242]] Reward -6.817039948146014 Loss 0.017883965745568275\n",
      "Episode 9 Step 1923 Action [[1.57239345]] Reward -7.417748215488083 Loss 0.07827433943748474\n",
      "Episode 9 Step 1924 Action [[1.0768242]] Reward -8.243445478193461 Loss 0.021278347820043564\n",
      "Episode 9 Step 1925 Action [[0.92353957]] Reward -9.202204892411935 Loss 1.6084282398223877\n",
      "Episode 9 Step 1926 Action [[0.96027238]] Reward -10.254936891814443 Loss 0.06086081638932228\n",
      "Episode 9 Step 1927 Action [[1.20051468]] Reward -10.371180109082191 Loss 0.1402512490749359\n",
      "Episode 9 Step 1928 Action [[1.11169413]] Reward -9.473403001741325 Loss 0.09478926658630371\n",
      "Episode 9 Step 1929 Action [[0.93829504]] Reward -8.54635066031096 Loss 0.07456181198358536\n",
      "Episode 9 Step 1930 Action [[0.93973531]] Reward -7.622880083856744 Loss 0.05790098011493683\n",
      "Episode 9 Step 1931 Action [[0.78417204]] Reward -6.755377651530788 Loss 0.05743106082081795\n",
      "Episode 9 Step 1932 Action [[0.73195825]] Reward -5.971176818847593 Loss 0.07266910374164581\n",
      "Episode 9 Step 1933 Action [[0.79772376]] Reward -5.311181976686979 Loss 0.15186259150505066\n",
      "Episode 9 Step 1934 Action [[0.94844039]] Reward -4.801050267628753 Loss 0.0552683100104332\n",
      "Episode 9 Step 1935 Action [[0.91937403]] Reward -4.448433935744512 Loss 0.030422132462263107\n",
      "Episode 9 Step 1936 Action [[0.67286115]] Reward -4.252059476303168 Loss 0.038181811571121216\n",
      "Episode 9 Step 1937 Action [[0.62969722]] Reward -4.225672580166462 Loss 0.04537336155772209\n",
      "Episode 9 Step 1938 Action [[0.80151599]] Reward -4.381541707703228 Loss 0.05868937075138092\n",
      "Episode 9 Step 1939 Action [[0.85545006]] Reward -4.7088625970796105 Loss 0.07377800345420837\n",
      "Episode 9 Step 1940 Action [[1.18173339]] Reward -5.20122273250096 Loss 1.259013295173645\n",
      "Episode 9 Step 1941 Action [[1.16717878]] Reward -5.818752934753551 Loss 0.2476087212562561\n",
      "Episode 9 Step 1942 Action [[1.05670936]] Reward -6.559661581810998 Loss 1.3000874519348145\n",
      "Episode 9 Step 1943 Action [[1.21147844]] Reward -7.4139342836644895 Loss 0.09041665494441986\n",
      "Episode 9 Step 1944 Action [[1.0153848]] Reward -8.320934110296397 Loss 0.06781771034002304\n",
      "Episode 9 Step 1945 Action [[1.145214]] Reward -9.28149878216045 Loss 0.12868112325668335\n",
      "Episode 9 Step 1946 Action [[0.87307284]] Reward -10.217252446484482 Loss 0.11391501873731613\n",
      "Episode 9 Step 1947 Action [[1.22581622]] Reward -10.487702802802593 Loss 0.03247173875570297\n",
      "Episode 9 Step 1948 Action [[1.058105]] Reward -9.497560117054048 Loss 0.056899815797805786\n",
      "Episode 9 Step 1949 Action [[0.9124238]] Reward -8.60160335866674 Loss 0.03736002743244171\n",
      "Episode 9 Step 1950 Action [[1.09382746]] Reward -7.827000512601546 Loss 0.061430856585502625\n",
      "Episode 9 Step 1951 Action [[0.98948708]] Reward -7.191971176959614 Loss 0.058885280042886734\n",
      "Episode 9 Step 1952 Action [[0.80763153]] Reward -6.738901008724036 Loss 0.03365814685821533\n",
      "Episode 9 Step 1953 Action [[0.9395986]] Reward -6.478476865866602 Loss 0.02874305471777916\n",
      "Episode 9 Step 1954 Action [[0.94733758]] Reward -6.417660394070161 Loss 0.23524674773216248\n",
      "Episode 9 Step 1955 Action [[1.01443346]] Reward -6.566906168496132 Loss 0.06220557540655136\n",
      "Episode 9 Step 1956 Action [[0.85707019]] Reward -6.927878874747542 Loss 0.1986047774553299\n",
      "Episode 9 Step 1957 Action [[0.79766956]] Reward -7.476412851320308 Loss 0.08401873707771301\n",
      "Episode 9 Step 1958 Action [[0.43919444]] Reward -8.188130009808312 Loss 0.13561612367630005\n",
      "Episode 9 Step 1959 Action [[0.55797628]] Reward -8.995641628923075 Loss 0.08065090328454971\n",
      "Episode 9 Step 1960 Action [[0.37997588]] Reward -9.894775200917733 Loss 0.11505697667598724\n",
      "Episode 9 Step 1961 Action [[0.12616855]] Reward -10.419545308974067 Loss 1.4754726886749268\n",
      "Episode 9 Step 1962 Action [[0.26464298]] Reward -9.587125389626921 Loss 0.13708893954753876\n",
      "Episode 9 Step 1963 Action [[0.34167732]] Reward -8.77280038217375 Loss 0.017927872017025948\n",
      "Episode 9 Step 1964 Action [[0.46095812]] Reward -7.998523862641822 Loss 0.06550350040197372\n",
      "Episode 9 Step 1965 Action [[0.22929359]] Reward -7.291374872910959 Loss 0.12105849385261536\n",
      "Episode 9 Step 1966 Action [[0.45068963]] Reward -6.665793537480222 Loss 0.042872026562690735\n",
      "Episode 9 Step 1967 Action [[0.66034883]] Reward -6.165214764390425 Loss 0.04973609745502472\n",
      "Episode 9 Step 1968 Action [[0.87498372]] Reward -5.792920020869377 Loss 1.42082941532135\n",
      "Episode 9 Step 1969 Action [[0.92564822]] Reward -5.546906965714284 Loss 0.1714116632938385\n",
      "Episode 9 Step 1970 Action [[1.18411679]] Reward -5.4253015053724125 Loss 0.07467179000377655\n",
      "Episode 9 Step 1971 Action [[1.20079602]] Reward -5.42060424968104 Loss 0.03704732656478882\n",
      "Episode 9 Step 1972 Action [[1.26214238]] Reward -5.528242585546094 Loss 0.10720813274383545\n",
      "Episode 9 Step 1973 Action [[0.99999415]] Reward -5.741745378975051 Loss 0.08360086381435394\n",
      "Episode 9 Step 1974 Action [[1.30011826]] Reward -6.076159958663571 Loss 0.01991528458893299\n",
      "Episode 9 Step 1975 Action [[1.3802999]] Reward -6.49348722763463 Loss 0.046750783920288086\n",
      "Episode 9 Step 1976 Action [[1.71769691]] Reward -6.981533714036897 Loss 0.11590950936079025\n",
      "Episode 9 Step 1977 Action [[1.63029191]] Reward -7.491948721216467 Loss 0.039965640753507614\n",
      "Episode 9 Step 1978 Action [[1.42041843]] Reward -8.02936491310883 Loss 0.030641017481684685\n",
      "Episode 9 Step 1979 Action [[1.16900168]] Reward -8.58980930095358 Loss 0.061422377824783325\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 9 Step 1980 Action [[1.44653543]] Reward -9.166934396163645 Loss 0.05364718288183212\n",
      "Episode 9 Step 1981 Action [[1.58224933]] Reward -9.693736656319349 Loss 0.04229093715548515\n",
      "Episode 9 Step 1982 Action [[1.29217545]] Reward -10.152325399304944 Loss 0.033030133694410324\n",
      "Episode 9 Step 1983 Action [[1.32779536]] Reward -9.632369197096184 Loss 0.025960683822631836\n",
      "Episode 9 Step 1984 Action [[1.36516252]] Reward -9.188847148260916 Loss 0.019512999802827835\n",
      "Episode 9 Step 1985 Action [[1.37527462]] Reward -8.846414660813247 Loss 0.018822992220520973\n",
      "Episode 9 Step 1986 Action [[1.22614514]] Reward -8.622258486003886 Loss 0.011327402666211128\n",
      "Episode 9 Step 1987 Action [[1.25319754]] Reward -8.522995695920601 Loss 0.022934192791581154\n",
      "Episode 9 Step 1988 Action [[1.31070992]] Reward -8.550904208894123 Loss 0.01519733015447855\n",
      "Episode 9 Step 1989 Action [[1.44397295]] Reward -8.710016259731466 Loss 0.030808329582214355\n",
      "Episode 9 Step 1990 Action [[1.49894741]] Reward -9.003935132814375 Loss 0.020923631265759468\n",
      "Episode 9 Step 1991 Action [[1.5700661]] Reward -9.425219002465646 Loss 0.033818088471889496\n",
      "Episode 9 Step 1992 Action [[1.61387576]] Reward -9.961763450356637 Loss 0.016183195635676384\n",
      "Episode 9 Step 1993 Action [[1.55039304]] Reward -9.761175940218557 Loss 0.011219702661037445\n",
      "Episode 9 Step 1994 Action [[1.56312178]] Reward -9.247672113055295 Loss 0.014389564283192158\n",
      "Episode 9 Step 1995 Action [[1.39035646]] Reward -8.697297972152471 Loss 0.02227955497801304\n",
      "Episode 9 Step 1996 Action [[1.50729369]] Reward -8.125302504947713 Loss 0.012345651164650917\n",
      "Episode 9 Step 1997 Action [[1.73986119]] Reward -7.563465695727624 Loss 0.012347716838121414\n",
      "Episode 9 Step 1998 Action [[1.54288555]] Reward -7.027058099885961 Loss 0.014627080410718918\n",
      "Episode 9 Step 1999 Action [[1.36988096]] Reward -6.5190119261090445 Loss 0.014088863506913185\n",
      "Now we save model\n",
      "TOTAL REWARD @ 9-th Episode  : Reward -1476.7176196065952\n",
      "Total Step: 2000\n",
      "\n",
      "Episode : 10 Replay Buffer 2000\n",
      "[-0.4808422   0.87680715  0.34471728]\n",
      "Episode 10 Step 2000 Action [[-0.70920941]] Reward -4.307274630268282 Loss 0.012205343693494797\n",
      "Episode 10 Step 2001 Action [[-0.4403005]] Reward -4.563036235993592 Loss 0.015984104946255684\n",
      "Episode 10 Step 2002 Action [[-0.16882018]] Reward -5.015676757292388 Loss 0.011682683601975441\n",
      "Episode 10 Step 2003 Action [[0.06703551]] Reward -5.682930011110605 Loss 0.01748614013195038\n",
      "Episode 10 Step 2004 Action [[0.37671169]] Reward -6.571353272734702 Loss 0.014782535843551159\n",
      "Episode 10 Step 2005 Action [[0.80994607]] Reward -7.68646598008558 Loss 0.23576949536800385\n",
      "Episode 10 Step 2006 Action [[1.4627565]] Reward -9.033436083269244 Loss 0.01211268175393343\n",
      "Episode 10 Step 2007 Action [[1.28694275]] Reward -10.624974233820065 Loss 0.020424701273441315\n",
      "Episode 10 Step 2008 Action [[1.20877766]] Reward -11.515906405703005 Loss 0.011947957798838615\n",
      "Episode 10 Step 2009 Action [[1.33408388]] Reward -10.264789085102608 Loss 0.010763988830149174\n",
      "Episode 10 Step 2010 Action [[1.38568339]] Reward -8.986314386354094 Loss 0.028711840510368347\n",
      "Episode 10 Step 2011 Action [[1.37738947]] Reward -7.717941578796704 Loss 0.009196801111102104\n",
      "Episode 10 Step 2012 Action [[1.45120196]] Reward -6.5065172981017065 Loss 0.0273958258330822\n",
      "Episode 10 Step 2013 Action [[1.14204538]] Reward -5.40763484833615 Loss 0.01551348902285099\n",
      "Episode 10 Step 2014 Action [[0.7806104]] Reward -4.42677044436785 Loss 0.4709445834159851\n",
      "Episode 10 Step 2015 Action [[0.68577359]] Reward -3.6024957085393217 Loss 0.025403235107660294\n",
      "Episode 10 Step 2016 Action [[0.48039373]] Reward -2.9746244883342365 Loss 0.032549746334552765\n",
      "Episode 10 Step 2017 Action [[0.76214216]] Reward -2.542504326279886 Loss 0.01624779962003231\n",
      "Episode 10 Step 2018 Action [[0.90538857]] Reward -2.310973138694725 Loss 0.016694992780685425\n",
      "Episode 10 Step 2019 Action [[1.37728714]] Reward -2.2546169416367463 Loss 0.01648634485900402\n",
      "Episode 10 Step 2020 Action [[1.45131374]] Reward -2.3471156022276967 Loss 0.009002413600683212\n",
      "Episode 10 Step 2021 Action [[1.27495031]] Reward -2.5796129665009 Loss 0.00989513285458088\n",
      "Episode 10 Step 2022 Action [[1.2753988]] Reward -2.9740456848225056 Loss 0.02039123699069023\n",
      "Episode 10 Step 2023 Action [[1.22366111]] Reward -3.538863466405365 Loss 0.008959868922829628\n",
      "Episode 10 Step 2024 Action [[1.32308153]] Reward -4.290677064513101 Loss 0.009745441377162933\n",
      "Episode 10 Step 2025 Action [[1.31064494]] Reward -5.218022205474803 Loss 1.65740966796875\n",
      "Episode 10 Step 2026 Action [[1.31484096]] Reward -6.320137681222941 Loss 0.1484576016664505\n",
      "Episode 10 Step 2027 Action [[1.66062036]] Reward -7.568669403579263 Loss 0.07910596579313278\n",
      "Episode 10 Step 2028 Action [[1.38741781]] Reward -8.85456093724058 Loss 0.07032948732376099\n",
      "Episode 10 Step 2029 Action [[1.2478252]] Reward -10.204133541975533 Loss 0.5587066411972046\n",
      "Episode 10 Step 2030 Action [[1.35393522]] Reward -11.541060378207975 Loss 0.06030425429344177\n",
      "Episode 10 Step 2031 Action [[1.19883808]] Reward -10.68971411664656 Loss 0.026993578299880028\n",
      "Episode 10 Step 2032 Action [[0.65997352]] Reward -9.295623414433917 Loss 0.03956732153892517\n",
      "Episode 10 Step 2033 Action [[0.6659147]] Reward -8.04885297078192 Loss 0.04011866822838783\n",
      "Episode 10 Step 2034 Action [[-0.05621864]] Reward -6.934240183255738 Loss 0.0159839428961277\n",
      "Episode 10 Step 2035 Action [[-0.34379527]] Reward -6.032588681532949 Loss 0.03446759283542633\n",
      "Episode 10 Step 2036 Action [[-0.52637083]] Reward -5.308974264234364 Loss 0.048236530274152756\n",
      "Episode 10 Step 2037 Action [[-0.23640558]] Reward -4.761882370663466 Loss 0.02649901807308197\n",
      "Episode 10 Step 2038 Action [[-0.20431615]] Reward -4.391338871815034 Loss 0.03321783244609833\n",
      "Episode 10 Step 2039 Action [[-0.68671846]] Reward -4.225100735405084 Loss 1.997657060623169\n",
      "Episode 10 Step 2040 Action [[-0.73741971]] Reward -4.250219200619957 Loss 0.1989937424659729\n",
      "Episode 10 Step 2041 Action [[-0.66420605]] Reward -4.451720064768007 Loss 1.4989020824432373\n",
      "Episode 10 Step 2042 Action [[-0.27082581]] Reward -4.833634233270529 Loss 0.14822134375572205\n",
      "Episode 10 Step 2043 Action [[-0.05884041]] Reward -5.427103318344828 Loss 0.13482901453971863\n",
      "Episode 10 Step 2044 Action [[0.34300304]] Reward -6.236463156242703 Loss 0.07748965919017792\n",
      "Episode 10 Step 2045 Action [[0.88369214]] Reward -7.284194168179077 Loss 0.12051637470722198\n",
      "Episode 10 Step 2046 Action [[1.04667975]] Reward -8.590124146379516 Loss 0.08006434142589569\n",
      "Episode 10 Step 2047 Action [[0.72127233]] Reward -10.08566765699926 Loss 0.08914782106876373\n",
      "Episode 10 Step 2048 Action [[0.869487]] Reward -11.627294608879112 Loss 0.15817469358444214\n",
      "Episode 10 Step 2049 Action [[1.31590335]] Reward -10.624307334708245 Loss 0.08493687212467194\n",
      "Episode 10 Step 2050 Action [[1.02745475]] Reward -9.3885154402906 Loss 0.034789856523275375\n",
      "Episode 10 Step 2051 Action [[1.34384933]] Reward -8.114399784171859 Loss 1.4035454988479614\n",
      "Episode 10 Step 2052 Action [[1.22259778]] Reward -6.919865560912484 Loss 0.020569492131471634\n",
      "Episode 10 Step 2053 Action [[1.1353692]] Reward -5.803419112176503 Loss 0.13511669635772705\n",
      "Episode 10 Step 2054 Action [[0.9710126]] Reward -4.814928171398409 Loss 0.07191532105207443\n",
      "Episode 10 Step 2055 Action [[1.21272872]] Reward -3.9823266794158823 Loss 0.061078570783138275\n",
      "Episode 10 Step 2056 Action [[1.38466582]] Reward -3.3422284132013624 Loss 0.06959269940853119\n",
      "Episode 10 Step 2057 Action [[1.26509317]] Reward -2.8740176975058653 Loss 0.06756342202425003\n",
      "Episode 10 Step 2058 Action [[1.32860842]] Reward -2.558578780653824 Loss 0.03338034451007843\n",
      "Episode 10 Step 2059 Action [[0.9737225]] Reward -2.395133278248581 Loss 0.041501082479953766\n",
      "Episode 10 Step 2060 Action [[0.87597207]] Reward -2.3873602447259503 Loss 0.06323560327291489\n",
      "Episode 10 Step 2061 Action [[0.69895542]] Reward -2.5513026497278073 Loss 0.039066724479198456\n",
      "Episode 10 Step 2062 Action [[0.7446443]] Reward -2.907276590653801 Loss 0.04457477480173111\n",
      "Episode 10 Step 2063 Action [[0.90289142]] Reward -3.4626270242577712 Loss 0.054866973310709\n",
      "Episode 10 Step 2064 Action [[1.01649938]] Reward -4.213788281575431 Loss 0.026624120771884918\n",
      "Episode 10 Step 2065 Action [[0.91662654]] Reward -5.159011891951614 Loss 0.04140462726354599\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 10 Step 2066 Action [[1.21385425]] Reward -6.313784373301581 Loss 1.2517204284667969\n",
      "Episode 10 Step 2067 Action [[1.24628879]] Reward -7.594562695780313 Loss 0.09739081561565399\n",
      "Episode 10 Step 2068 Action [[1.29928692]] Reward -8.975677941581383 Loss 0.06677358597517014\n",
      "Episode 10 Step 2069 Action [[1.34377842]] Reward -10.384465753670003 Loss 0.05470413714647293\n",
      "Episode 10 Step 2070 Action [[1.38432575]] Reward -11.746584513630392 Loss 0.12260402739048004\n",
      "Episode 10 Step 2071 Action [[1.40748044]] Reward -10.669004360661075 Loss 0.07260282337665558\n",
      "Episode 10 Step 2072 Action [[1.71853964]] Reward -9.222055015560333 Loss 0.02648540958762169\n",
      "Episode 10 Step 2073 Action [[1.33696845]] Reward -7.879900367499548 Loss 0.0816967636346817\n",
      "Episode 10 Step 2074 Action [[0.44437885]] Reward -6.767552184216045 Loss 0.05197274684906006\n",
      "Episode 10 Step 2075 Action [[-0.35026999]] Reward -5.919928443430423 Loss 0.03173571452498436\n",
      "Episode 10 Step 2076 Action [[-0.47155761]] Reward -5.291818466019037 Loss 0.06350816786289215\n",
      "Episode 10 Step 2077 Action [[-0.33153264]] Reward -4.835621931542545 Loss 0.06561405956745148\n",
      "Episode 10 Step 2078 Action [[-0.58753742]] Reward -4.556820896884102 Loss 1.6082490682601929\n",
      "Episode 10 Step 2079 Action [[-0.52749913]] Reward -4.462932884581488 Loss 0.30914413928985596\n",
      "Episode 10 Step 2080 Action [[-0.6145919]] Reward -4.550355037892896 Loss 0.2699822187423706\n",
      "Episode 10 Step 2081 Action [[-0.72055828]] Reward -4.815868477244606 Loss 0.04773997887969017\n",
      "Episode 10 Step 2082 Action [[-0.8076199]] Reward -5.2483435237216 Loss 0.16034725308418274\n",
      "Episode 10 Step 2083 Action [[-0.57683936]] Reward -5.833240775652011 Loss 0.41274261474609375\n",
      "Episode 10 Step 2084 Action [[-0.17321997]] Reward -6.583881881148634 Loss 0.03668662905693054\n",
      "Episode 10 Step 2085 Action [[0.27267467]] Reward -7.516659777456067 Loss 0.17278803884983063\n",
      "Episode 10 Step 2086 Action [[0.5848389]] Reward -8.634783784406954 Loss 0.08278562128543854\n",
      "Episode 10 Step 2087 Action [[1.01251709]] Reward -9.905985484787006 Loss 0.0541493222117424\n",
      "Episode 10 Step 2088 Action [[1.22405792]] Reward -11.313224904030871 Loss 0.08707846701145172\n",
      "Episode 10 Step 2089 Action [[1.21745957]] Reward -10.359530807163202 Loss 0.13270819187164307\n",
      "Episode 10 Step 2090 Action [[0.9524629]] Reward -9.218994940205906 Loss 0.043166838586330414\n",
      "Episode 10 Step 2091 Action [[0.59195703]] Reward -8.049153119734191 Loss 0.09126002341508865\n",
      "Episode 10 Step 2092 Action [[0.51878805]] Reward -6.903691664495223 Loss 0.08246029168367386\n",
      "Episode 10 Step 2093 Action [[0.49430516]] Reward -5.869649479472118 Loss 0.0449526347219944\n",
      "Episode 10 Step 2094 Action [[0.34234419]] Reward -4.991786175124261 Loss 0.027326686307787895\n",
      "Episode 10 Step 2095 Action [[0.28390159]] Reward -4.292386237791011 Loss 0.05353038012981415\n",
      "Episode 10 Step 2096 Action [[0.4120455]] Reward -3.797976914609272 Loss 0.09273932129144669\n",
      "Episode 10 Step 2097 Action [[0.6398724]] Reward -3.5174479043161964 Loss 0.02648269198834896\n",
      "Episode 10 Step 2098 Action [[0.85323968]] Reward -3.439001620683306 Loss 0.07044675946235657\n",
      "Episode 10 Step 2099 Action [[0.95332921]] Reward -3.5454215683742567 Loss 0.04051920771598816\n",
      "Episode 10 Step 2100 Action [[0.84731611]] Reward -3.828013177042558 Loss 0.02678721584379673\n",
      "Episode 10 Step 2101 Action [[0.86183167]] Reward -4.297469656995895 Loss 0.02551349624991417\n",
      "Episode 10 Step 2102 Action [[0.83981376]] Reward -4.950568816569619 Loss 0.04233189672231674\n",
      "Episode 10 Step 2103 Action [[1.07165955]] Reward -5.784313415714956 Loss 0.025209100916981697\n",
      "Episode 10 Step 2104 Action [[1.27874301]] Reward -6.749891383171233 Loss 0.0425192229449749\n",
      "Episode 10 Step 2105 Action [[1.60301028]] Reward -7.802780098337534 Loss 0.04274245351552963\n",
      "Episode 10 Step 2106 Action [[1.70461001]] Reward -8.86737913053607 Loss 0.036442019045352936\n",
      "Episode 10 Step 2107 Action [[2.00098944]] Reward -9.911352437886018 Loss 0.019231125712394714\n",
      "Episode 10 Step 2108 Action [[2.06757557]] Reward -10.851140840112805 Loss 1.9299628734588623\n",
      "Episode 10 Step 2109 Action [[1.49774292]] Reward -10.11354774909485 Loss 0.03304092958569527\n",
      "Episode 10 Step 2110 Action [[1.29435434]] Reward -9.078920127013465 Loss 1.6643781661987305\n",
      "Episode 10 Step 2111 Action [[0.61198308]] Reward -8.174876441684438 Loss 0.0854959636926651\n",
      "Episode 10 Step 2112 Action [[-0.01508309]] Reward -7.438747688800673 Loss 0.1010008156299591\n",
      "Episode 10 Step 2113 Action [[0.04758312]] Reward -6.851731398122702 Loss 0.1146930381655693\n",
      "Episode 10 Step 2114 Action [[-0.20979023]] Reward -6.391492567767827 Loss 0.06805570423603058\n",
      "Episode 10 Step 2115 Action [[-0.32334486]] Reward -6.077520587748911 Loss 0.11699096858501434\n",
      "Episode 10 Step 2116 Action [[-0.55508429]] Reward -5.9062474298918515 Loss 0.09403540194034576\n",
      "Episode 10 Step 2117 Action [[-0.76354977]] Reward -5.8720239994143375 Loss 0.14581003785133362\n",
      "Episode 10 Step 2118 Action [[-0.72148951]] Reward -5.961037276415663 Loss 0.1095031201839447\n",
      "Episode 10 Step 2119 Action [[-0.67918068]] Reward -6.171675266723659 Loss 0.052237432450056076\n",
      "Episode 10 Step 2120 Action [[-0.66409227]] Reward -6.5013613181548955 Loss 0.04397832602262497\n",
      "Episode 10 Step 2121 Action [[-0.71233168]] Reward -6.940761090884613 Loss 1.7995561361312866\n",
      "Episode 10 Step 2122 Action [[-0.30751806]] Reward -7.469459611569588 Loss 0.1308738887310028\n",
      "Episode 10 Step 2123 Action [[0.43926619]] Reward -8.110404291100897 Loss 0.2760802209377289\n",
      "Episode 10 Step 2124 Action [[0.78193474]] Reward -8.903433336699337 Loss 0.1831381916999817\n",
      "Episode 10 Step 2125 Action [[0.6097645]] Reward -9.818943990791846 Loss 1.7142807245254517\n",
      "Episode 10 Step 2126 Action [[0.80575028]] Reward -10.508656547411134 Loss 0.09179334342479706\n",
      "Episode 10 Step 2127 Action [[0.73344532]] Reward -9.689553613982204 Loss 0.023533407598733902\n",
      "Episode 10 Step 2128 Action [[0.74160014]] Reward -8.8487903375321 Loss 0.07482945919036865\n",
      "Episode 10 Step 2129 Action [[0.66589966]] Reward -8.02258625239779 Loss 0.07063828408718109\n",
      "Episode 10 Step 2130 Action [[0.49806111]] Reward -7.238886925135334 Loss 0.030301012098789215\n",
      "Episode 10 Step 2131 Action [[0.56388121]] Reward -6.530157880395135 Loss 0.07324831187725067\n",
      "Episode 10 Step 2132 Action [[0.9776468]] Reward -5.938900438322391 Loss 0.11048494279384613\n",
      "Episode 10 Step 2133 Action [[0.81679605]] Reward -5.482738138064353 Loss 0.04026421532034874\n",
      "Episode 10 Step 2134 Action [[0.78009791]] Reward -5.150938730828312 Loss 0.09747345000505447\n",
      "Episode 10 Step 2135 Action [[0.62653722]] Reward -4.962450394298008 Loss 1.5886989831924438\n",
      "Episode 10 Step 2136 Action [[1.00365459]] Reward -4.930639892967517 Loss 0.04542272165417671\n",
      "Episode 10 Step 2137 Action [[0.91615653]] Reward -5.043180189892561 Loss 0.14871510863304138\n",
      "Episode 10 Step 2138 Action [[0.6104066]] Reward -5.29918366315795 Loss 0.10098380595445633\n",
      "Episode 10 Step 2139 Action [[0.59855662]] Reward -5.717849668022713 Loss 0.08856022357940674\n",
      "Episode 10 Step 2140 Action [[0.78772497]] Reward -6.286779405197853 Loss 0.07346074283123016\n",
      "Episode 10 Step 2141 Action [[0.9400149]] Reward -6.969386343218816 Loss 0.05924607440829277\n",
      "Episode 10 Step 2142 Action [[1.13413437]] Reward -7.732513819795649 Loss 0.08854886889457703\n",
      "Episode 10 Step 2143 Action [[1.54245203]] Reward -8.531414658605428 Loss 0.16001370549201965\n",
      "Episode 10 Step 2144 Action [[1.35087318]] Reward -9.292285914761953 Loss 0.01921909861266613\n",
      "Episode 10 Step 2145 Action [[1.37001709]] Reward -10.0393474220607 Loss 0.06261035799980164\n",
      "Episode 10 Step 2146 Action [[1.5253234]] Reward -10.201701102394779 Loss 0.12489266693592072\n",
      "Episode 10 Step 2147 Action [[1.18318344]] Reward -9.42076227332794 Loss 0.03952071815729141\n",
      "Episode 10 Step 2148 Action [[0.9382631]] Reward -8.749770819642569 Loss 0.09233196079730988\n",
      "Episode 10 Step 2149 Action [[0.79327109]] Reward -8.19382493007071 Loss 0.049866270273923874\n",
      "Episode 10 Step 2150 Action [[0.54831291]] Reward -7.76029491593738 Loss 0.03224039077758789\n",
      "Episode 10 Step 2151 Action [[0.58174551]] Reward -7.456786401858779 Loss 0.04756128042936325\n",
      "Episode 10 Step 2152 Action [[0.97171741]] Reward -7.2866941013621895 Loss 0.11582421511411667\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 10 Step 2153 Action [[1.45004334]] Reward -7.275310213853802 Loss 0.03691532462835312\n",
      "Episode 10 Step 2154 Action [[1.4903218]] Reward -7.460933884857607 Loss 0.09098824858665466\n",
      "Episode 10 Step 2155 Action [[1.56117186]] Reward -7.849474654701155 Loss 0.054184094071388245\n",
      "Episode 10 Step 2156 Action [[1.7552846]] Reward -8.433411424888384 Loss 0.043752022087574005\n",
      "Episode 10 Step 2157 Action [[1.90451379]] Reward -9.207719535582829 Loss 0.05817446857690811\n",
      "Episode 10 Step 2158 Action [[1.80331665]] Reward -10.149340435284067 Loss 0.09731075912714005\n",
      "Episode 10 Step 2159 Action [[1.9314008]] Reward -10.127313499711732 Loss 0.05592774227261543\n",
      "Episode 10 Step 2160 Action [[1.54623517]] Reward -9.356748525813792 Loss 2.1534717082977295\n",
      "Episode 10 Step 2161 Action [[1.78289419]] Reward -8.521406838984966 Loss 0.5068925619125366\n",
      "Episode 10 Step 2162 Action [[1.42030861]] Reward -7.69076810542854 Loss 0.27245989441871643\n",
      "Episode 10 Step 2163 Action [[1.72679163]] Reward -6.858960698143107 Loss 0.14816522598266602\n",
      "Episode 10 Step 2164 Action [[1.73680809]] Reward -6.101389088990744 Loss 2.22139048576355\n",
      "Episode 10 Step 2165 Action [[1.66973868]] Reward -5.417326579961428 Loss 0.055893175303936005\n",
      "Episode 10 Step 2166 Action [[1.60649463]] Reward -4.824973456982728 Loss 0.11112158000469208\n",
      "Episode 10 Step 2167 Action [[1.50879123]] Reward -4.343071716081948 Loss 0.770846962928772\n",
      "Episode 10 Step 2168 Action [[1.40582305]] Reward -3.984628414485877 Loss 1.3655292987823486\n",
      "Episode 10 Step 2169 Action [[1.13440825]] Reward -3.7607706178889866 Loss 0.0765213668346405\n",
      "Episode 10 Step 2170 Action [[0.74009328]] Reward -3.6855738860455447 Loss 0.11423951387405396\n",
      "Episode 10 Step 2171 Action [[0.82160217]] Reward -3.7875568537349564 Loss 0.1259925365447998\n",
      "Episode 10 Step 2172 Action [[0.81276293]] Reward -4.0711190773178645 Loss 0.07263031601905823\n",
      "Episode 10 Step 2173 Action [[0.44007388]] Reward -4.537204132024935 Loss 0.6263599395751953\n",
      "Episode 10 Step 2174 Action [[0.67622328]] Reward -5.220723238613021 Loss 0.10196810960769653\n",
      "Episode 10 Step 2175 Action [[0.45402087]] Reward -6.073051916174057 Loss 0.0784558355808258\n",
      "Episode 10 Step 2176 Action [[0.59885354]] Reward -7.114969335121651 Loss 0.032338276505470276\n",
      "Episode 10 Step 2177 Action [[0.9134127]] Reward -8.275305007850815 Loss 0.12379497289657593\n",
      "Episode 10 Step 2178 Action [[0.7174315]] Reward -9.466014354166745 Loss 0.040525585412979126\n",
      "Episode 10 Step 2179 Action [[0.66123287]] Reward -10.69373138627935 Loss 0.6970453858375549\n",
      "Episode 10 Step 2180 Action [[0.8664129]] Reward -10.84918852529874 Loss 0.15149015188217163\n",
      "Episode 10 Step 2181 Action [[0.88435078]] Reward -9.61545849663598 Loss 0.14997020363807678\n",
      "Episode 10 Step 2182 Action [[1.01805484]] Reward -8.451652189034318 Loss 0.10421987622976303\n",
      "Episode 10 Step 2183 Action [[0.72614393]] Reward -7.403088899880435 Loss 0.8334252834320068\n",
      "Episode 10 Step 2184 Action [[0.67231597]] Reward -6.5420611781097575 Loss 0.06256125867366791\n",
      "Episode 10 Step 2185 Action [[0.36905122]] Reward -5.875637610841443 Loss 0.025879710912704468\n",
      "Episode 10 Step 2186 Action [[0.41129849]] Reward -5.425477274326741 Loss 1.6183629035949707\n",
      "Episode 10 Step 2187 Action [[0.23464377]] Reward -5.186436153843971 Loss 0.15494830906391144\n",
      "Episode 10 Step 2188 Action [[0.25293589]] Reward -5.163774129080866 Loss 0.08799725770950317\n",
      "Episode 10 Step 2189 Action [[0.23588916]] Reward -5.3553855621016 Loss 0.043991897255182266\n",
      "Episode 10 Step 2190 Action [[0.94311226]] Reward -5.759215290184415 Loss 0.17232131958007812\n",
      "Episode 10 Step 2191 Action [[1.57479091]] Reward -6.432210133793961 Loss 0.20499059557914734\n",
      "Episode 10 Step 2192 Action [[1.52152075]] Reward -7.413967167275543 Loss 0.11100344359874725\n",
      "Episode 10 Step 2193 Action [[1.23183542]] Reward -8.6411012832461 Loss 0.5112287402153015\n",
      "Episode 10 Step 2194 Action [[1.23691761]] Reward -10.020238563964195 Loss 0.12350883334875107\n",
      "Episode 10 Step 2195 Action [[1.32674082]] Reward -11.496930319764662 Loss 0.10407422482967377\n",
      "Episode 10 Step 2196 Action [[0.94506173]] Reward -10.374273172251808 Loss 0.15397042036056519\n",
      "Episode 10 Step 2197 Action [[1.1876546]] Reward -9.169092448624681 Loss 0.360343337059021\n",
      "Episode 10 Step 2198 Action [[1.41933205]] Reward -7.991555402477371 Loss 0.12718147039413452\n",
      "Episode 10 Step 2199 Action [[1.34487752]] Reward -6.876620676200615 Loss 0.7455229163169861\n",
      "TOTAL REWARD @ 10-th Episode  : Reward -1352.858381056485\n",
      "Total Step: 2200\n",
      "\n",
      "Episode : 11 Replay Buffer 2200\n",
      "[0.72162271 0.69228654 0.26911235]\n",
      "Episode 11 Step 2200 Action [[-0.50678553]] Reward -0.592192988474333 Loss 0.21176959574222565\n",
      "Episode 11 Step 2201 Action [[-0.91960221]] Reward -0.6920135584611575 Loss 0.09777365624904633\n",
      "Episode 11 Step 2202 Action [[-1.06703599]] Reward -0.8574657248316037 Loss 0.07088323682546616\n",
      "Episode 11 Step 2203 Action [[-1.06606121]] Reward -1.1001432411320446 Loss 0.13681554794311523\n",
      "Episode 11 Step 2204 Action [[-0.87076794]] Reward -1.4459606970617327 Loss 0.1675129234790802\n",
      "Episode 11 Step 2205 Action [[-0.93470239]] Reward -1.943261659134065 Loss 0.6975388526916504\n",
      "Episode 11 Step 2206 Action [[-0.99952054]] Reward -2.613956831122164 Loss 0.15174561738967896\n",
      "Episode 11 Step 2207 Action [[-1.06390545]] Reward -3.494206031208258 Loss 0.16607503592967987\n",
      "Episode 11 Step 2208 Action [[-0.7321532]] Reward -4.61434074259495 Loss 0.10379956662654877\n",
      "Episode 11 Step 2209 Action [[-0.67739975]] Reward -6.05813164514542 Loss 1.5045926570892334\n",
      "Episode 11 Step 2210 Action [[0.50478452]] Reward -7.791403312473638 Loss 0.06935996562242508\n",
      "Episode 11 Step 2211 Action [[1.27339544]] Reward -10.011738486131307 Loss 0.19354301691055298\n",
      "Episode 11 Step 2212 Action [[1.44808348]] Reward -12.627041126448061 Loss 0.1650589108467102\n",
      "Episode 11 Step 2213 Action [[1.50223032]] Reward -14.61720200154616 Loss 0.16224020719528198\n",
      "Episode 11 Step 2214 Action [[1.30178655]] Reward -12.749475999625886 Loss 0.07000732421875\n",
      "Episode 11 Step 2215 Action [[1.4538951]] Reward -10.750167397814028 Loss 0.11491502076387405\n",
      "Episode 11 Step 2216 Action [[1.24877812]] Reward -8.798940448427084 Loss 0.08693566918373108\n",
      "Episode 11 Step 2217 Action [[0.51028942]] Reward -6.9565175319854005 Loss 0.05794409289956093\n",
      "Episode 11 Step 2218 Action [[-0.41990536]] Reward -5.257115544517946 Loss 0.055279575288295746\n",
      "Episode 11 Step 2219 Action [[-0.40527925]] Reward -3.7695853886686344 Loss 0.04923124983906746\n",
      "Episode 11 Step 2220 Action [[-0.59583243]] Reward -2.6489405244189004 Loss 0.08268088847398758\n",
      "Episode 11 Step 2221 Action [[-0.76704647]] Reward -1.8147202250246828 Loss 0.09622468054294586\n",
      "Episode 11 Step 2222 Action [[-0.35020802]] Reward -1.210790227509022 Loss 0.1403079330921173\n",
      "Episode 11 Step 2223 Action [[-0.72450417]] Reward -0.8205337180263744 Loss 0.0435737669467926\n",
      "Episode 11 Step 2224 Action [[-0.54917471]] Reward -0.5317445003461357 Loss 0.06498152762651443\n",
      "Episode 11 Step 2225 Action [[-0.64649278]] Reward -0.34483048161018265 Loss 0.09097567200660706\n",
      "Episode 11 Step 2226 Action [[-0.56306194]] Reward -0.2161843532978305 Loss 0.04472283273935318\n",
      "Episode 11 Step 2227 Action [[-0.42238458]] Reward -0.1336227367923134 Loss 0.05191081762313843\n",
      "Episode 11 Step 2228 Action [[-0.16499544]] Reward -0.0826625233881311 Loss 0.684092104434967\n",
      "Episode 11 Step 2229 Action [[-0.21751719]] Reward -0.053580629844158935 Loss 0.074835866689682\n",
      "Episode 11 Step 2230 Action [[-0.56537839]] Reward -0.03454437297805153 Loss 0.06817092001438141\n",
      "Episode 11 Step 2231 Action [[-0.32368529]] Reward -0.02143004327424055 Loss 0.09478685259819031\n",
      "Episode 11 Step 2232 Action [[-0.15427214]] Reward -0.017985130962778578 Loss 0.05796163156628609\n",
      "Episode 11 Step 2233 Action [[0.28312407]] Reward -0.020241051919156462 Loss 0.06666906923055649\n",
      "Episode 11 Step 2234 Action [[0.15116701]] Reward -0.0240039244923232 Loss 0.028145335614681244\n",
      "Episode 11 Step 2235 Action [[0.48392573]] Reward -0.03146199420399635 Loss 0.03916309028863907\n",
      "Episode 11 Step 2236 Action [[0.33257445]] Reward -0.0386419425800248 Loss 0.03896774724125862\n",
      "Episode 11 Step 2237 Action [[0.47146949]] Reward -0.05078310440339074 Loss 0.037425633519887924\n",
      "Episode 11 Step 2238 Action [[0.44515983]] Reward -0.06563367030924638 Loss 0.724565863609314\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 11 Step 2239 Action [[0.80546086]] Reward -0.08731728353159289 Loss 0.4059751629829407\n",
      "Episode 11 Step 2240 Action [[0.83735672]] Reward -0.10909859254059362 Loss 0.05586736276745796\n",
      "Episode 11 Step 2241 Action [[1.14312239]] Reward -0.138338834481614 Loss 0.07640734314918518\n",
      "Episode 11 Step 2242 Action [[1.32974221]] Reward -0.16938462955506522 Loss 0.8976768851280212\n",
      "Episode 11 Step 2243 Action [[1.77562458]] Reward -0.20606605462697 Loss 0.033278536051511765\n",
      "Episode 11 Step 2244 Action [[1.70081308]] Reward -0.23866143671998774 Loss 0.0469394251704216\n",
      "Episode 11 Step 2245 Action [[1.90308141]] Reward -0.2838071216148413 Loss 0.07092016190290451\n",
      "Episode 11 Step 2246 Action [[1.68310722]] Reward -0.33337144418875003 Loss 1.9556784629821777\n",
      "Episode 11 Step 2247 Action [[1.85868672]] Reward -0.4065693668877369 Loss 0.07763437926769257\n",
      "Episode 11 Step 2248 Action [[1.85412991]] Reward -0.49459951320824785 Loss 0.053503748029470444\n",
      "Episode 11 Step 2249 Action [[1.83686324]] Reward -0.6100422203202389 Loss 0.08033736050128937\n",
      "Episode 11 Step 2250 Action [[1.74996591]] Reward -0.7631387723754088 Loss 0.7313606142997742\n",
      "Episode 11 Step 2251 Action [[1.58138843]] Reward -0.9717717808466425 Loss 3.900829792022705\n",
      "Episode 11 Step 2252 Action [[1.84433498]] Reward -1.264627142195807 Loss 0.4072226285934448\n",
      "Episode 11 Step 2253 Action [[2.04259228]] Reward -1.634528606158241 Loss 0.11914744973182678\n",
      "Episode 11 Step 2254 Action [[1.97679422]] Reward -2.105629738104641 Loss 0.08413520455360413\n",
      "Episode 11 Step 2255 Action [[2.00761821]] Reward -2.71852866073511 Loss 0.18887655436992645\n",
      "Episode 11 Step 2256 Action [[1.78607494]] Reward -3.4956822252993915 Loss 0.2637006938457489\n",
      "Episode 11 Step 2257 Action [[2.34473925]] Reward -4.494874498958678 Loss 0.23889616131782532\n",
      "Episode 11 Step 2258 Action [[1.94257783]] Reward -5.670211204297643 Loss 0.12027565389871597\n",
      "Episode 11 Step 2259 Action [[2.1740533]] Reward -7.0414182644556575 Loss 0.2465551644563675\n",
      "Episode 11 Step 2260 Action [[2.11869422]] Reward -8.545422202323623 Loss 0.2381960153579712\n",
      "Episode 11 Step 2261 Action [[2.14558014]] Reward -10.115994808425617 Loss 0.19215241074562073\n",
      "Episode 11 Step 2262 Action [[1.92165763]] Reward -11.66188903837215 Loss 0.18822498619556427\n",
      "Episode 11 Step 2263 Action [[1.83063536]] Reward -12.094312844574395 Loss 0.08213484287261963\n",
      "Episode 11 Step 2264 Action [[1.45153721]] Reward -10.319287772776157 Loss 0.21370552480220795\n",
      "Episode 11 Step 2265 Action [[1.45842395]] Reward -8.69587322709409 Loss 0.08711148798465729\n",
      "Episode 11 Step 2266 Action [[0.96145686]] Reward -7.238676182117353 Loss 0.05033329129219055\n",
      "Episode 11 Step 2267 Action [[-0.23773336]] Reward -6.047855069576738 Loss 0.8336673974990845\n",
      "Episode 11 Step 2268 Action [[-0.69163726]] Reward -5.158863660614651 Loss 0.03888170048594475\n",
      "Episode 11 Step 2269 Action [[-0.7730235]] Reward -4.476743275725299 Loss 0.14205427467823029\n",
      "Episode 11 Step 2270 Action [[-0.94256939]] Reward -3.974864261299633 Loss 0.13496175408363342\n",
      "Episode 11 Step 2271 Action [[-0.91125332]] Reward -3.6528677417783313 Loss 0.07466793805360794\n",
      "Episode 11 Step 2272 Action [[-1.07091926]] Reward -3.5039235312936032 Loss 0.03714535012841225\n",
      "Episode 11 Step 2273 Action [[-1.14641696]] Reward -3.524894565974213 Loss 0.07596024125814438\n",
      "Episode 11 Step 2274 Action [[-0.85000767]] Reward -3.7084413836081875 Loss 0.07046429067850113\n",
      "Episode 11 Step 2275 Action [[-1.21105844]] Reward -4.077781298924576 Loss 0.09087636321783066\n",
      "Episode 11 Step 2276 Action [[-1.35642883]] Reward -4.600026799152908 Loss 0.06562691926956177\n",
      "Episode 11 Step 2277 Action [[-1.05865563]] Reward -5.264378634210658 Loss 0.038134507834911346\n",
      "Episode 11 Step 2278 Action [[-1.02807843]] Reward -6.101752276131941 Loss 0.052359241992235184\n",
      "Episode 11 Step 2279 Action [[-0.23175602]] Reward -7.077243070780129 Loss 0.062064409255981445\n",
      "Episode 11 Step 2280 Action [[0.46006827]] Reward -8.271527373728384 Loss 0.06732174754142761\n",
      "Episode 11 Step 2281 Action [[1.01526908]] Reward -9.680995644119081 Loss 0.023704636842012405\n",
      "Episode 11 Step 2282 Action [[1.0433532]] Reward -11.270329184430782 Loss 0.03513322398066521\n",
      "Episode 11 Step 2283 Action [[0.89571866]] Reward -11.02294599313118 Loss 0.037896957248449326\n",
      "Episode 11 Step 2284 Action [[0.74908134]] Reward -9.726835028412316 Loss 0.10489125549793243\n",
      "Episode 11 Step 2285 Action [[1.05399946]] Reward -8.40948640060929 Loss 0.03673543408513069\n",
      "Episode 11 Step 2286 Action [[0.8610072]] Reward -7.171704278849925 Loss 0.05552078038454056\n",
      "Episode 11 Step 2287 Action [[0.52075825]] Reward -6.009730974451895 Loss 0.04041808843612671\n",
      "Episode 11 Step 2288 Action [[-0.1344438]] Reward -4.970606688340496 Loss 0.03876553848385811\n",
      "Episode 11 Step 2289 Action [[-0.26112342]] Reward -4.093993904532619 Loss 0.03283967077732086\n",
      "Episode 11 Step 2290 Action [[0.04599674]] Reward -3.4627565607675606 Loss 0.023571912199258804\n",
      "Episode 11 Step 2291 Action [[0.3528121]] Reward -3.091339641011341 Loss 0.018557662144303322\n",
      "Episode 11 Step 2292 Action [[0.8385856]] Reward -2.9499248861955283 Loss 0.03199753165245056\n",
      "Episode 11 Step 2293 Action [[0.86931938]] Reward -3.0037933598846025 Loss 0.052648402750492096\n",
      "Episode 11 Step 2294 Action [[0.82806933]] Reward -3.240286351613185 Loss 0.020637378096580505\n",
      "Episode 11 Step 2295 Action [[1.24321783]] Reward -3.6684610735614753 Loss 0.04922695830464363\n",
      "Episode 11 Step 2296 Action [[1.17513085]] Reward -4.253460411733383 Loss 0.028863947838544846\n",
      "Episode 11 Step 2297 Action [[1.30446323]] Reward -5.016934905357193 Loss 0.045720599591732025\n",
      "Episode 11 Step 2298 Action [[1.39797167]] Reward -5.931550198699071 Loss 0.03904770687222481\n",
      "Episode 11 Step 2299 Action [[1.58977816]] Reward -6.972078552197862 Loss 0.013821267522871494\n",
      "Episode 11 Step 2300 Action [[1.70604293]] Reward -8.080903624610583 Loss 0.017913712188601494\n",
      "Episode 11 Step 2301 Action [[1.97085696]] Reward -9.210352207304703 Loss 0.024485688656568527\n",
      "Episode 11 Step 2302 Action [[1.97715835]] Reward -10.275469748355613 Loss 0.018876448273658752\n",
      "Episode 11 Step 2303 Action [[1.96090233]] Reward -11.106145029615185 Loss 0.01533443946391344\n",
      "Episode 11 Step 2304 Action [[1.94546594]] Reward -9.886490375945225 Loss 0.03305322676897049\n",
      "Episode 11 Step 2305 Action [[1.38236809]] Reward -8.785972381413073 Loss 0.014242220669984818\n",
      "Episode 11 Step 2306 Action [[0.92890898]] Reward -7.873486122735436 Loss 0.03216402977705002\n",
      "Episode 11 Step 2307 Action [[0.38692019]] Reward -7.143460597752746 Loss 0.009171909652650356\n",
      "Episode 11 Step 2308 Action [[-0.19211416]] Reward -6.5930966709664505 Loss 2.9438040256500244\n",
      "Episode 11 Step 2309 Action [[-0.22544327]] Reward -6.202962360898839 Loss 0.17364993691444397\n",
      "Episode 11 Step 2310 Action [[-0.37908364]] Reward -5.953820775996955 Loss 0.3786185681819916\n",
      "Episode 11 Step 2311 Action [[-0.49337057]] Reward -5.849404186221935 Loss 0.22256168723106384\n",
      "Episode 11 Step 2312 Action [[-0.25492668]] Reward -5.884346920408176 Loss 0.2618548274040222\n",
      "Episode 11 Step 2313 Action [[-0.32623518]] Reward -6.070237722728426 Loss 0.23706047236919403\n",
      "Episode 11 Step 2314 Action [[0.21293878]] Reward -6.399569271241836 Loss 0.09333045780658722\n",
      "Episode 11 Step 2315 Action [[0.91955173]] Reward -6.908805213109075 Loss 0.2905260920524597\n",
      "Episode 11 Step 2316 Action [[1.19846815]] Reward -7.6414308516336575 Loss 0.7492440938949585\n",
      "Episode 11 Step 2317 Action [[1.38669878]] Reward -8.581806751379862 Loss 0.2776712477207184\n",
      "Episode 11 Step 2318 Action [[1.26595735]] Reward -9.701810198062146 Loss 0.3415122330188751\n",
      "Episode 11 Step 2319 Action [[1.24782366]] Reward -10.921571663635714 Loss 0.49863946437835693\n",
      "Episode 11 Step 2320 Action [[1.4795471]] Reward -10.095623057823955 Loss 0.8740331530570984\n",
      "Episode 11 Step 2321 Action [[1.31235385]] Reward -9.119531808522611 Loss 0.33054572343826294\n",
      "Episode 11 Step 2322 Action [[1.22820845]] Reward -8.11274248344581 Loss 0.6000616550445557\n",
      "Episode 11 Step 2323 Action [[0.87949335]] Reward -7.1247844531534525 Loss 0.10023130476474762\n",
      "Episode 11 Step 2324 Action [[0.43566262]] Reward -6.1842339327163245 Loss 0.2421383410692215\n",
      "Episode 11 Step 2325 Action [[0.73620039]] Reward -5.344661922356343 Loss 0.2676777243614197\n",
      "Episode 11 Step 2326 Action [[0.90685537]] Reward -4.685584556147754 Loss 0.3091750144958496\n",
      "Episode 11 Step 2327 Action [[0.19909655]] Reward -4.197142031388953 Loss 0.2616581916809082\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 11 Step 2328 Action [[0.31108028]] Reward -3.878253351214211 Loss 0.31356489658355713\n",
      "Episode 11 Step 2329 Action [[0.67379239]] Reward -3.77406224644471 Loss 0.15530040860176086\n",
      "Episode 11 Step 2330 Action [[0.69702526]] Reward -3.8661054680058857 Loss 0.2746724784374237\n",
      "Episode 11 Step 2331 Action [[0.82474025]] Reward -4.146397398718462 Loss 0.303068608045578\n",
      "Episode 11 Step 2332 Action [[0.89560248]] Reward -4.605268364599444 Loss 0.13065578043460846\n",
      "Episode 11 Step 2333 Action [[1.24023251]] Reward -5.235369425980264 Loss 0.06200264394283295\n",
      "Episode 11 Step 2334 Action [[1.06364364]] Reward -5.9894416524936505 Loss 0.15378034114837646\n",
      "Episode 11 Step 2335 Action [[1.2827083]] Reward -6.888404483372857 Loss 0.23524747788906097\n",
      "Episode 11 Step 2336 Action [[1.31275043]] Reward -7.860227500435733 Loss 0.2074413001537323\n",
      "Episode 11 Step 2337 Action [[1.13489875]] Reward -8.877234420663115 Loss 0.147599995136261\n",
      "Episode 11 Step 2338 Action [[1.16795966]] Reward -9.92355658333186 Loss 0.14371484518051147\n",
      "Episode 11 Step 2339 Action [[1.25424039]] Reward -10.929643205727011 Loss 0.1335301697254181\n",
      "Episode 11 Step 2340 Action [[1.30147001]] Reward -10.007747786839579 Loss 0.07047280669212341\n",
      "Episode 11 Step 2341 Action [[0.9872723]] Reward -8.970192780249516 Loss 0.07987891137599945\n",
      "Episode 11 Step 2342 Action [[0.47745493]] Reward -8.061623866617145 Loss 0.19640877842903137\n",
      "Episode 11 Step 2343 Action [[-0.2301204]] Reward -7.304957747455551 Loss 0.2384982854127884\n",
      "Episode 11 Step 2344 Action [[-0.41806915]] Reward -6.698931201435143 Loss 0.12112436443567276\n",
      "Episode 11 Step 2345 Action [[-0.42863041]] Reward -6.215232308548321 Loss 0.15544307231903076\n",
      "Episode 11 Step 2346 Action [[-0.38003509]] Reward -5.859766512669321 Loss 2.6479783058166504\n",
      "Episode 11 Step 2347 Action [[-0.09405536]] Reward -5.645640161859059 Loss 0.4286026060581207\n",
      "Episode 11 Step 2348 Action [[-0.03762894]] Reward -5.592534398536231 Loss 1.0078771114349365\n",
      "Episode 11 Step 2349 Action [[-0.0097995]] Reward -5.715782448397474 Loss 0.11491818726062775\n",
      "Episode 11 Step 2350 Action [[0.16708081]] Reward -6.01674786961968 Loss 0.4391377866268158\n",
      "Episode 11 Step 2351 Action [[-0.21903617]] Reward -6.502566985634241 Loss 0.6373673677444458\n",
      "Episode 11 Step 2352 Action [[-0.25834579]] Reward -7.119327642911697 Loss 0.1810447871685028\n",
      "Episode 11 Step 2353 Action [[-0.05517724]] Reward -7.852207904103002 Loss 0.5518043041229248\n",
      "Episode 11 Step 2354 Action [[0.45760975]] Reward -8.698094950702432 Loss 0.26292282342910767\n",
      "Episode 11 Step 2355 Action [[0.73808088]] Reward -9.674845019805282 Loss 0.22463051974773407\n",
      "Episode 11 Step 2356 Action [[0.88815607]] Reward -10.743277342131261 Loss 0.16335654258728027\n",
      "Episode 11 Step 2357 Action [[1.2521294]] Reward -9.967297681428098 Loss 0.6077128648757935\n",
      "Episode 11 Step 2358 Action [[1.32233044]] Reward -9.065874424856279 Loss 0.22676223516464233\n",
      "Episode 11 Step 2359 Action [[1.20784079]] Reward -8.155906172792282 Loss 0.10165093839168549\n",
      "Episode 11 Step 2360 Action [[1.0274595]] Reward -7.25806867680126 Loss 0.21301904320716858\n",
      "Episode 11 Step 2361 Action [[0.97064918]] Reward -6.40968161288317 Loss 0.5242021083831787\n",
      "Episode 11 Step 2362 Action [[0.6731905]] Reward -5.657025502463842 Loss 0.20743392407894135\n",
      "Episode 11 Step 2363 Action [[0.64826357]] Reward -5.0233662914629384 Loss 0.47682854533195496\n",
      "Episode 11 Step 2364 Action [[0.73496391]] Reward -4.550896160625012 Loss 0.11372095346450806\n",
      "Episode 11 Step 2365 Action [[0.37882997]] Reward -4.2509590519137195 Loss 0.1561417132616043\n",
      "Episode 11 Step 2366 Action [[0.56940858]] Reward -4.133350156108367 Loss 0.31107771396636963\n",
      "Episode 11 Step 2367 Action [[0.77106045]] Reward -4.208299558045366 Loss 0.28015682101249695\n",
      "Episode 11 Step 2368 Action [[0.98520089]] Reward -4.460604014267741 Loss 0.10797025263309479\n",
      "Episode 11 Step 2369 Action [[0.74524359]] Reward -4.871025335434721 Loss 0.19602471590042114\n",
      "Episode 11 Step 2370 Action [[0.86342634]] Reward -5.46003497855607 Loss 0.21304933726787567\n",
      "Episode 11 Step 2371 Action [[1.07625844]] Reward -6.1972714610001525 Loss 0.12700307369232178\n",
      "Episode 11 Step 2372 Action [[0.75289174]] Reward -7.0408784519001095 Loss 0.2968935966491699\n",
      "Episode 11 Step 2373 Action [[0.98055423]] Reward -8.01747204657956 Loss 0.25756025314331055\n",
      "Episode 11 Step 2374 Action [[0.90214091]] Reward -9.030784917394948 Loss 0.06510437279939651\n",
      "Episode 11 Step 2375 Action [[0.72789703]] Reward -10.065813575310516 Loss 2.9319663047790527\n",
      "Episode 11 Step 2376 Action [[0.50752715]] Reward -10.877287221612152 Loss 0.11107011139392853\n",
      "Episode 11 Step 2377 Action [[-0.00547859]] Reward -9.828938473850823 Loss 0.24756214022636414\n",
      "Episode 11 Step 2378 Action [[-0.54560978]] Reward -8.84516939388135 Loss 0.43813884258270264\n",
      "Episode 11 Step 2379 Action [[-0.65424614]] Reward -7.938305825138449 Loss 0.1266687959432602\n",
      "Episode 11 Step 2380 Action [[-0.7735125]] Reward -7.096809373153019 Loss 0.12426766753196716\n",
      "Episode 11 Step 2381 Action [[-0.77614335]] Reward -6.347527105399267 Loss 0.12762600183486938\n",
      "Episode 11 Step 2382 Action [[-0.87392359]] Reward -5.708077621725468 Loss 0.2900346517562866\n",
      "Episode 11 Step 2383 Action [[-1.03986143]] Reward -5.201079259044251 Loss 0.5753403902053833\n",
      "Episode 11 Step 2384 Action [[-1.09103665]] Reward -4.834313718891786 Loss 0.18979521095752716\n",
      "Episode 11 Step 2385 Action [[-1.15203364]] Reward -4.606999945516759 Loss 0.08115219324827194\n",
      "Episode 11 Step 2386 Action [[-1.13467356]] Reward -4.520544388640771 Loss 0.0785183310508728\n",
      "Episode 11 Step 2387 Action [[-1.39067782]] Reward -4.577216036048369 Loss 0.0938352569937706\n",
      "Episode 11 Step 2388 Action [[-1.18453987]] Reward -4.762335895426558 Loss 0.2063368260860443\n",
      "Episode 11 Step 2389 Action [[-0.64723712]] Reward -5.086795970144456 Loss 3.0398175716400146\n",
      "Episode 11 Step 2390 Action [[-0.55371578]] Reward -5.589158627130496 Loss 0.4076964259147644\n",
      "Episode 11 Step 2391 Action [[-0.46282107]] Reward -6.2569005225308025 Loss 0.3794344961643219\n",
      "Episode 11 Step 2392 Action [[-0.48224738]] Reward -7.078571831494545 Loss 0.0645611360669136\n",
      "Episode 11 Step 2393 Action [[-0.57182169]] Reward -8.01853543082462 Loss 0.16249337792396545\n",
      "Episode 11 Step 2394 Action [[-0.5821227]] Reward -9.027118913812616 Loss 2.577775716781616\n",
      "Episode 11 Step 2395 Action [[0.45949097]] Reward -10.06459757051962 Loss 0.11145657300949097\n",
      "Episode 11 Step 2396 Action [[0.70315037]] Reward -10.854543240356612 Loss 0.3381240665912628\n",
      "Episode 11 Step 2397 Action [[0.81116916]] Reward -9.850238919251135 Loss 0.2721995413303375\n",
      "Episode 11 Step 2398 Action [[0.95368539]] Reward -8.836929880533985 Loss 0.975177526473999\n",
      "Episode 11 Step 2399 Action [[0.87859547]] Reward -7.848812198181309 Loss 0.06910157203674316\n",
      "TOTAL REWARD @ 11-th Episode  : Reward -1124.2710013406754\n",
      "Total Step: 2400\n",
      "\n",
      "Episode : 12 Replay Buffer 2400\n",
      "[ 0.84761663 -0.53060913  0.36677723]\n",
      "Episode 12 Step 2400 Action [[1.07271469]] Reward -0.3274410641355131 Loss 0.20282627642154694\n",
      "Episode 12 Step 2401 Action [[1.14866658]] Reward -0.3086263151020055 Loss 0.040943752974271774\n",
      "Episode 12 Step 2402 Action [[1.05787408]] Reward -0.31268166064980846 Loss 0.0852942019701004\n",
      "Episode 12 Step 2403 Action [[1.01769722]] Reward -0.3412963589501769 Loss 0.14405247569084167\n",
      "Episode 12 Step 2404 Action [[1.39742535]] Reward -0.39987904766121285 Loss 0.07502917945384979\n",
      "Episode 12 Step 2405 Action [[1.28155348]] Reward -0.47931430366984185 Loss 0.08846953511238098\n",
      "Episode 12 Step 2406 Action [[1.35693829]] Reward -0.5981934605247029 Loss 0.11677204072475433\n",
      "Episode 12 Step 2407 Action [[1.03873647]] Reward -0.7611811045435591 Loss 0.6228321194648743\n",
      "Episode 12 Step 2408 Action [[1.08739965]] Reward -1.0078735449155962 Loss 0.0766485333442688\n",
      "Episode 12 Step 2409 Action [[0.71815283]] Reward -1.3473898845396095 Loss 2.603400945663452\n",
      "Episode 12 Step 2410 Action [[0.97751465]] Reward -1.8488455497926652 Loss 0.34530359506607056\n",
      "Episode 12 Step 2411 Action [[0.91878797]] Reward -2.5050870972326065 Loss 0.1899287849664688\n",
      "Episode 12 Step 2412 Action [[0.97238963]] Reward -3.3842409315946966 Loss 0.16222994029521942\n",
      "Episode 12 Step 2413 Action [[0.88385037]] Reward -4.51418229779974 Loss 0.27192145586013794\n",
      "Episode 12 Step 2414 Action [[0.75772125]] Reward -5.941170456841882 Loss 0.1478537917137146\n",
      "Episode 12 Step 2415 Action [[0.8367757]] Reward -7.6759247787449665 Loss 0.07163824886083603\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 12 Step 2416 Action [[0.80016283]] Reward -9.6336808222696 Loss 0.39767515659332275\n",
      "Episode 12 Step 2417 Action [[0.99546023]] Reward -11.733418810606404 Loss 0.11522817611694336\n",
      "Episode 12 Step 2418 Action [[1.09538712]] Reward -13.788092436786465 Loss 0.046170517802238464\n",
      "Episode 12 Step 2419 Action [[1.19332061]] Reward -12.249455615102526 Loss 0.36555182933807373\n",
      "Episode 12 Step 2420 Action [[1.35774033]] Reward -10.15996009411939 Loss 2.4085299968719482\n",
      "Episode 12 Step 2421 Action [[1.09941304]] Reward -8.166555845924606 Loss 1.3688710927963257\n",
      "Episode 12 Step 2422 Action [[-0.29472438]] Reward -6.4320367941767085 Loss 0.5716749429702759\n",
      "Episode 12 Step 2423 Action [[-0.36511249]] Reward -5.110501567002916 Loss 0.6076379418373108\n",
      "Episode 12 Step 2424 Action [[-0.9439689]] Reward -4.024795429753794 Loss 0.2919372022151947\n",
      "Episode 12 Step 2425 Action [[-0.9138159]] Reward -3.2079687763349676 Loss 0.5449816584587097\n",
      "Episode 12 Step 2426 Action [[-0.80723933]] Reward -2.5850075612333705 Loss 0.22030794620513916\n",
      "Episode 12 Step 2427 Action [[-0.9654552]] Reward -2.142408517175215 Loss 0.18929420411586761\n",
      "Episode 12 Step 2428 Action [[-0.8669285]] Reward -1.8728185952459493 Loss 2.809330701828003\n",
      "Episode 12 Step 2429 Action [[-0.6587421]] Reward -1.755218934265944 Loss 0.5083145499229431\n",
      "Episode 12 Step 2430 Action [[-0.7092832]] Reward -1.7949680670545545 Loss 2.227219343185425\n",
      "Episode 12 Step 2431 Action [[-0.97672001]] Reward -1.998678982954477 Loss 0.11479741334915161\n",
      "Episode 12 Step 2432 Action [[-0.86090328]] Reward -2.3588191783844614 Loss 2.5328280925750732\n",
      "Episode 12 Step 2433 Action [[-1.13931487]] Reward -2.9074376301710325 Loss 3.01800799369812\n",
      "Episode 12 Step 2434 Action [[-1.01413355]] Reward -3.631371173499069 Loss 0.6867577433586121\n",
      "Episode 12 Step 2435 Action [[-0.95640065]] Reward -4.577108976775114 Loss 2.000459671020508\n",
      "Episode 12 Step 2436 Action [[-0.81534996]] Reward -5.751176837658425 Loss 0.4795384705066681\n",
      "Episode 12 Step 2437 Action [[-0.34016562]] Reward -7.157961469877431 Loss 0.29735684394836426\n",
      "Episode 12 Step 2438 Action [[0.6430166]] Reward -8.829772684661654 Loss 0.2820265293121338\n",
      "Episode 12 Step 2439 Action [[0.97747559]] Reward -10.836238490264671 Loss 2.7139930725097656\n",
      "Episode 12 Step 2440 Action [[0.85524574]] Reward -13.013759979749146 Loss 0.21370351314544678\n",
      "Episode 12 Step 2441 Action [[0.91171438]] Reward -11.96857359121468 Loss 0.573642373085022\n",
      "Episode 12 Step 2442 Action [[0.84784448]] Reward -10.270530195677631 Loss 0.34438222646713257\n",
      "Episode 12 Step 2443 Action [[1.06247698]] Reward -8.56340554395704 Loss 0.34816408157348633\n",
      "Episode 12 Step 2444 Action [[0.88875821]] Reward -6.971990398386682 Loss 0.5446995496749878\n",
      "Episode 12 Step 2445 Action [[0.32784127]] Reward -5.514940783374289 Loss 0.34880879521369934\n",
      "Episode 12 Step 2446 Action [[-0.37332999]] Reward -4.216792926279006 Loss 0.5632392764091492\n",
      "Episode 12 Step 2447 Action [[-0.02910022]] Reward -3.125329412810917 Loss 2.6833620071411133\n",
      "Episode 12 Step 2448 Action [[-0.19842166]] Reward -2.343350696437188 Loss 3.461124897003174\n",
      "Episode 12 Step 2449 Action [[0.25629853]] Reward -1.7930029531119303 Loss 2.6283679008483887\n",
      "Episode 12 Step 2450 Action [[0.87146677]] Reward -1.4676444890078324 Loss 0.5711992979049683\n",
      "Episode 12 Step 2451 Action [[0.38222891]] Reward -1.3086557486130799 Loss 0.27334821224212646\n",
      "Episode 12 Step 2452 Action [[0.7648461]] Reward -1.2838893359892252 Loss 0.7486189603805542\n",
      "Episode 12 Step 2453 Action [[0.64764891]] Reward -1.3932030262103872 Loss 0.2915976643562317\n",
      "Episode 12 Step 2454 Action [[0.66813656]] Reward -1.6464100770896015 Loss 0.318062961101532\n",
      "Episode 12 Step 2455 Action [[0.43480168]] Reward -2.0602967698526276 Loss 0.4094199538230896\n",
      "Episode 12 Step 2456 Action [[0.77276323]] Reward -2.6856247543907386 Loss 0.3796881437301636\n",
      "Episode 12 Step 2457 Action [[0.70811937]] Reward -3.50696092158656 Loss 0.252460777759552\n",
      "Episode 12 Step 2458 Action [[0.99181553]] Reward -4.581466172437592 Loss 0.22916045784950256\n",
      "Episode 12 Step 2459 Action [[1.21937149]] Reward -5.873611240529386 Loss 0.16841420531272888\n",
      "Episode 12 Step 2460 Action [[0.96035422]] Reward -7.355275002140206 Loss 0.25458449125289917\n",
      "Episode 12 Step 2461 Action [[1.18335857]] Reward -9.056588588674419 Loss 4.624121189117432\n",
      "Episode 12 Step 2462 Action [[1.24299937]] Reward -10.813488868256341 Loss 0.5484809875488281\n",
      "Episode 12 Step 2463 Action [[1.25993362]] Reward -12.543920236340451 Loss 0.727883517742157\n",
      "Episode 12 Step 2464 Action [[1.31584286]] Reward -11.698911984462258 Loss 0.3577437698841095\n",
      "Episode 12 Step 2465 Action [[1.55359576]] Reward -9.883091161512583 Loss 0.22591398656368256\n",
      "Episode 12 Step 2466 Action [[1.2926812]] Reward -8.154239497059647 Loss 2.0342485904693604\n",
      "Episode 12 Step 2467 Action [[-0.23822681]] Reward -6.658730410701279 Loss 0.5833745002746582\n",
      "Episode 12 Step 2468 Action [[-0.54662766]] Reward -5.526870467712069 Loss 0.14474403858184814\n",
      "Episode 12 Step 2469 Action [[-0.87540285]] Reward -4.606471522243781 Loss 0.9597025513648987\n",
      "Episode 12 Step 2470 Action [[-1.24854879]] Reward -3.8950073198507438 Loss 3.339564085006714\n",
      "Episode 12 Step 2471 Action [[-1.19935738]] Reward -3.3747746370601464 Loss 0.22143131494522095\n",
      "Episode 12 Step 2472 Action [[-1.27075738]] Reward -3.015612949139942 Loss 0.20670953392982483\n",
      "Episode 12 Step 2473 Action [[-1.52105028]] Reward -2.817764113957749 Loss 3.1841490268707275\n",
      "Episode 12 Step 2474 Action [[-1.58866113]] Reward -2.7684822083203895 Loss 0.06018904969096184\n",
      "Episode 12 Step 2475 Action [[-1.57296667]] Reward -2.856866246349891 Loss 0.3354277014732361\n",
      "Episode 12 Step 2476 Action [[-1.37071935]] Reward -3.084953903375554 Loss 0.4383222460746765\n",
      "Episode 12 Step 2477 Action [[-1.56985889]] Reward -3.473674247089827 Loss 0.3334793746471405\n",
      "Episode 12 Step 2478 Action [[-1.42105578]] Reward -4.006637828328995 Loss 1.3654677867889404\n",
      "Episode 12 Step 2479 Action [[-1.25037534]] Reward -4.7084237767787425 Loss 0.1804143339395523\n",
      "Episode 12 Step 2480 Action [[-1.02139501]] Reward -5.5889126664269 Loss 0.07109613716602325\n",
      "Episode 12 Step 2481 Action [[-0.97873948]] Reward -6.654368258646457 Loss 0.2456669807434082\n",
      "Episode 12 Step 2482 Action [[-0.27353003]] Reward -7.861653497337563 Loss 0.14693285524845123\n",
      "Episode 12 Step 2483 Action [[0.73058807]] Reward -9.276618452743897 Loss 0.15683582425117493\n",
      "Episode 12 Step 2484 Action [[1.04752966]] Reward -10.940413841336031 Loss 4.315278053283691\n",
      "Episode 12 Step 2485 Action [[1.06270133]] Reward -11.784995739022909 Loss 0.129624143242836\n",
      "Episode 12 Step 2486 Action [[0.93982171]] Reward -10.422368092399362 Loss 1.2536903619766235\n",
      "Episode 12 Step 2487 Action [[0.72248916]] Reward -9.010464198830627 Loss 1.1507797241210938\n",
      "Episode 12 Step 2488 Action [[0.60424449]] Reward -7.60365569607528 Loss 0.3483606278896332\n",
      "Episode 12 Step 2489 Action [[0.57944984]] Reward -6.286898834006956 Loss 0.42290425300598145\n",
      "Episode 12 Step 2490 Action [[0.07362447]] Reward -5.128356750436471 Loss 2.979463815689087\n",
      "Episode 12 Step 2491 Action [[-0.0973395]] Reward -4.134862611721634 Loss 0.7935917973518372\n",
      "Episode 12 Step 2492 Action [[0.07787382]] Reward -3.3761908298745684 Loss 0.17969287931919098\n",
      "Episode 12 Step 2493 Action [[0.35505607]] Reward -2.8727658598952788 Loss 3.493635892868042\n",
      "Episode 12 Step 2494 Action [[0.71631557]] Reward -2.6019781821524446 Loss 0.45529037714004517\n",
      "Episode 12 Step 2495 Action [[1.10808427]] Reward -2.532875525928762 Loss 0.7793865203857422\n",
      "Episode 12 Step 2496 Action [[1.18359859]] Reward -2.63317303498831 Loss 0.4802173376083374\n",
      "Episode 12 Step 2497 Action [[0.92701883]] Reward -2.8940628956384953 Loss 0.2551136612892151\n",
      "Episode 12 Step 2498 Action [[0.83423947]] Reward -3.343467147544435 Loss 0.3563903570175171\n",
      "Episode 12 Step 2499 Action [[1.18788161]] Reward -3.9971105890569003 Loss 0.4561958312988281\n",
      "Episode 12 Step 2500 Action [[1.24977671]] Reward -4.8169876547287 Loss 0.3646537661552429\n",
      "Episode 12 Step 2501 Action [[1.30010032]] Reward -5.809851318539906 Loss 0.18263915181159973\n",
      "Episode 12 Step 2502 Action [[1.1116495]] Reward -6.9530338255147415 Loss 0.24302253127098083\n",
      "Episode 12 Step 2503 Action [[1.10723551]] Reward -8.244486165026084 Loss 0.09515149891376495\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 12 Step 2504 Action [[0.89457334]] Reward -9.609321828067255 Loss 0.10852466523647308\n",
      "Episode 12 Step 2505 Action [[1.16167829]] Reward -11.018502702588293 Loss 0.3428477942943573\n",
      "Episode 12 Step 2506 Action [[0.97598475]] Reward -11.302112594245205 Loss 0.12060566991567612\n",
      "Episode 12 Step 2507 Action [[0.86212247]] Reward -9.884428837980808 Loss 1.2266899347305298\n",
      "Episode 12 Step 2508 Action [[0.31122162]] Reward -8.546656700149086 Loss 0.12252147495746613\n",
      "Episode 12 Step 2509 Action [[0.43877875]] Reward -7.369965228744796 Loss 0.3248424530029297\n",
      "Episode 12 Step 2510 Action [[-0.11472282]] Reward -6.328953930180036 Loss 0.1369142383337021\n",
      "Episode 12 Step 2511 Action [[-0.92618839]] Reward -5.502417412393876 Loss 0.11652861535549164\n",
      "Episode 12 Step 2512 Action [[-1.26668333]] Reward -4.880345689117789 Loss 0.056235551834106445\n",
      "Episode 12 Step 2513 Action [[-1.36655492]] Reward -4.41331924137777 Loss 1.0784200429916382\n",
      "Episode 12 Step 2514 Action [[-1.23360723]] Reward -4.086238386999678 Loss 0.08659593760967255\n",
      "Episode 12 Step 2515 Action [[-1.35166324]] Reward -3.901240390124474 Loss 0.1527257114648819\n",
      "Episode 12 Step 2516 Action [[-1.2638299]] Reward -3.8624811346786903 Loss 0.2921156883239746\n",
      "Episode 12 Step 2517 Action [[-1.31110185]] Reward -3.972489424474542 Loss 0.12536844611167908\n",
      "Episode 12 Step 2518 Action [[-1.10749009]] Reward -4.229809016644089 Loss 0.0930284932255745\n",
      "Episode 12 Step 2519 Action [[-1.08973604]] Reward -4.650022230369334 Loss 0.30121296644210815\n",
      "Episode 12 Step 2520 Action [[-0.08303984]] Reward -5.226902579938088 Loss 0.14208272099494934\n",
      "Episode 12 Step 2521 Action [[0.80970996]] Reward -6.062441933910281 Loss 0.13573062419891357\n",
      "Episode 12 Step 2522 Action [[1.0195672]] Reward -7.21048844748565 Loss 0.16875603795051575\n",
      "Episode 12 Step 2523 Action [[0.98748101]] Reward -8.615772302361103 Loss 0.144980788230896\n",
      "Episode 12 Step 2524 Action [[1.07461186]] Reward -10.19769008178512 Loss 0.03893033415079117\n",
      "Episode 12 Step 2525 Action [[1.37483974]] Reward -11.897587174321494 Loss 0.0976124033331871\n",
      "Episode 12 Step 2526 Action [[1.18078409]] Reward -10.858612388483456 Loss 0.08748762309551239\n",
      "Episode 12 Step 2527 Action [[1.30685596]] Reward -9.511684532922883 Loss 0.12497995793819427\n",
      "Episode 12 Step 2528 Action [[1.19781047]] Reward -8.170984251187132 Loss 0.09778544306755066\n",
      "Episode 12 Step 2529 Action [[1.18295276]] Reward -6.866322935310617 Loss 0.1484258472919464\n",
      "Episode 12 Step 2530 Action [[1.06205294]] Reward -5.6681190966430055 Loss 0.09088209271430969\n",
      "Episode 12 Step 2531 Action [[0.37683346]] Reward -4.611244085362397 Loss 0.058195218443870544\n",
      "Episode 12 Step 2532 Action [[0.20888766]] Reward -3.6955172045430276 Loss 0.07760575413703918\n",
      "Episode 12 Step 2533 Action [[-0.2543283]] Reward -3.000760847836013 Loss 4.151217937469482\n",
      "Episode 12 Step 2534 Action [[-0.02220043]] Reward -2.5281973346244597 Loss 0.2082182765007019\n",
      "Episode 12 Step 2535 Action [[0.38933036]] Reward -2.3043464984343327 Loss 0.2001926153898239\n",
      "Episode 12 Step 2536 Action [[0.93358381]] Reward -2.297177544544327 Loss 3.423248767852783\n",
      "Episode 12 Step 2537 Action [[1.12518613]] Reward -2.465789267203027 Loss 0.4665296673774719\n",
      "Episode 12 Step 2538 Action [[1.13040151]] Reward -2.7966977295962407 Loss 0.18575483560562134\n",
      "Episode 12 Step 2539 Action [[0.89614188]] Reward -3.301376635482076 Loss 3.9542081356048584\n",
      "Episode 12 Step 2540 Action [[0.88646351]] Reward -4.016955864574255 Loss 1.2983285188674927\n",
      "Episode 12 Step 2541 Action [[1.0511871]] Reward -4.944159934811735 Loss 0.28987693786621094\n",
      "Episode 12 Step 2542 Action [[1.12203656]] Reward -6.055788839671593 Loss 2.514360189437866\n",
      "Episode 12 Step 2543 Action [[1.23320104]] Reward -7.331098183500256 Loss 0.6509255170822144\n",
      "Episode 12 Step 2544 Action [[1.08402636]] Reward -8.711768735365446 Loss 0.5076159834861755\n",
      "Episode 12 Step 2545 Action [[1.3153493]] Reward -10.173747598723176 Loss 0.7107090950012207\n",
      "Episode 12 Step 2546 Action [[1.17936491]] Reward -11.578133698117876 Loss 0.3764113485813141\n",
      "Episode 12 Step 2547 Action [[1.19879809]] Reward -11.00801211012451 Loss 0.17006459832191467\n",
      "Episode 12 Step 2548 Action [[1.44330012]] Reward -9.524455100365874 Loss 0.24841119349002838\n",
      "Episode 12 Step 2549 Action [[1.20089388]] Reward -8.124548183022613 Loss 0.3857210874557495\n",
      "Episode 12 Step 2550 Action [[0.86207829]] Reward -6.9225189121548585 Loss 0.39346620440483093\n",
      "Episode 12 Step 2551 Action [[-0.65814832]] Reward -5.9555937191866715 Loss 0.42329394817352295\n",
      "Episode 12 Step 2552 Action [[-1.07585303]] Reward -5.2649625149564585 Loss 1.8345178365707397\n",
      "Episode 12 Step 2553 Action [[-1.30738896]] Reward -4.737185784224759 Loss 0.07854770869016647\n",
      "Episode 12 Step 2554 Action [[-1.74494395]] Reward -4.3553591027889595 Loss 0.22070884704589844\n",
      "Episode 12 Step 2555 Action [[-1.74676104]] Reward -4.107218965365033 Loss 0.4978269934654236\n",
      "Episode 12 Step 2556 Action [[-1.38784264]] Reward -3.975324877436945 Loss 0.26855581998825073\n",
      "Episode 12 Step 2557 Action [[-0.83013009]] Reward -3.973987056413264 Loss 0.1643337458372116\n",
      "Episode 12 Step 2558 Action [[-1.370437]] Reward -4.142977295112297 Loss 0.2244628518819809\n",
      "Episode 12 Step 2559 Action [[-1.11141096]] Reward -4.450694926095857 Loss 0.21325647830963135\n",
      "Episode 12 Step 2560 Action [[-0.60066477]] Reward -4.919205677893349 Loss 1.3404262065887451\n",
      "Episode 12 Step 2561 Action [[0.2120357]] Reward -5.587905361667515 Loss 0.2490701675415039\n",
      "Episode 12 Step 2562 Action [[0.86424451]] Reward -6.522246193479087 Loss 0.30440405011177063\n",
      "Episode 12 Step 2563 Action [[1.38724349]] Reward -7.748563390729801 Loss 0.1515602022409439\n",
      "Episode 12 Step 2564 Action [[1.20109294]] Reward -9.262098030926383 Loss 0.11247362941503525\n",
      "Episode 12 Step 2565 Action [[1.09311111]] Reward -10.919911964897967 Loss 0.1306384652853012\n",
      "Episode 12 Step 2566 Action [[1.12861233]] Reward -11.555842149311342 Loss 0.3381633162498474\n",
      "Episode 12 Step 2567 Action [[1.03662762]] Reward -10.24238511472759 Loss 3.576542854309082\n",
      "Episode 12 Step 2568 Action [[1.25715356]] Reward -8.886279930090668 Loss 4.162761688232422\n",
      "Episode 12 Step 2569 Action [[0.71304403]] Reward -7.575593704262802 Loss 1.6768927574157715\n",
      "Episode 12 Step 2570 Action [[0.02617191]] Reward -6.2881182951692765 Loss 1.1400940418243408\n",
      "Episode 12 Step 2571 Action [[-0.45505344]] Reward -5.1014391650539 Loss 0.40416067838668823\n",
      "Episode 12 Step 2572 Action [[-0.46942861]] Reward -4.1166201520638355 Loss 0.3730160593986511\n",
      "Episode 12 Step 2573 Action [[-0.00548931]] Reward -3.406159980254111 Loss 0.5711402297019958\n",
      "Episode 12 Step 2574 Action [[0.2387959]] Reward -2.975904291606049 Loss 0.18751820921897888\n",
      "Episode 12 Step 2575 Action [[0.94807058]] Reward -2.782095330744813 Loss 0.14752475917339325\n",
      "Episode 12 Step 2576 Action [[1.38730524]] Reward -2.7861710742598573 Loss 0.4790777266025543\n",
      "Episode 12 Step 2577 Action [[1.82394553]] Reward -2.945337798274786 Loss 0.4408732056617737\n",
      "Episode 12 Step 2578 Action [[1.70339105]] Reward -3.2289102250446287 Loss 0.18682876229286194\n",
      "Episode 12 Step 2579 Action [[1.66927436]] Reward -3.6551778950307865 Loss 0.257477343082428\n",
      "Episode 12 Step 2580 Action [[1.77712316]] Reward -4.229180264342601 Loss 0.1981019377708435\n",
      "Episode 12 Step 2581 Action [[1.30392981]] Reward -4.935994617370383 Loss 0.2488391101360321\n",
      "Episode 12 Step 2582 Action [[1.18180727]] Reward -5.833672602721852 Loss 0.06078457459807396\n",
      "Episode 12 Step 2583 Action [[0.98862233]] Reward -6.889256355605901 Loss 0.12657636404037476\n",
      "Episode 12 Step 2584 Action [[1.10775622]] Reward -8.088896560377451 Loss 0.4242757558822632\n",
      "Episode 12 Step 2585 Action [[1.57390395]] Reward -9.345729773219386 Loss 0.27516186237335205\n",
      "Episode 12 Step 2586 Action [[1.40409897]] Reward -10.531201270706491 Loss 0.1438053697347641\n",
      "Episode 12 Step 2587 Action [[1.11737479]] Reward -11.18288507184416 Loss 1.364933967590332\n",
      "Episode 12 Step 2588 Action [[1.24138421]] Reward -9.904456213193738 Loss 0.16377560794353485\n",
      "Episode 12 Step 2589 Action [[0.70635245]] Reward -8.69018447351635 Loss 0.060772985219955444\n",
      "Episode 12 Step 2590 Action [[0.67890253]] Reward -7.637206937200806 Loss 0.1569204032421112\n",
      "Episode 12 Step 2591 Action [[-0.16040268]] Reward -6.731695703474109 Loss 0.24445180594921112\n",
      "Episode 12 Step 2592 Action [[-0.43492004]] Reward -6.030554734610705 Loss 0.1472397744655609\n",
      "Episode 12 Step 2593 Action [[-0.68459579]] Reward -5.488995314222718 Loss 1.1371681690216064\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 12 Step 2594 Action [[-0.95989513]] Reward -5.104472427159833 Loss 0.14355838298797607\n",
      "Episode 12 Step 2595 Action [[-0.80046018]] Reward -4.868572417916017 Loss 0.14425478875637054\n",
      "Episode 12 Step 2596 Action [[-1.03385216]] Reward -4.78041329630689 Loss 0.1638737916946411\n",
      "Episode 12 Step 2597 Action [[-0.82401647]] Reward -4.837778136759845 Loss 4.136962413787842\n",
      "Episode 12 Step 2598 Action [[-0.66101133]] Reward -5.048787021717604 Loss 0.45375385880470276\n",
      "Episode 12 Step 2599 Action [[-0.19744752]] Reward -5.4233249525298195 Loss 3.5034031867980957\n",
      "Now we save model\n",
      "TOTAL REWARD @ 12-th Episode  : Reward -1139.4072896943387\n",
      "Total Step: 2600\n",
      "\n",
      "Episode : 13 Replay Buffer 2600\n",
      "[-0.52512282 -0.85102646 -0.65522599]\n",
      "Episode 13 Step 2600 Action [[0.78229551]] Reward -4.553457830238518 Loss 0.4510763883590698\n",
      "Episode 13 Step 2601 Action [[0.76002688]] Reward -4.902057032651268 Loss 0.5533158779144287\n",
      "Episode 13 Step 2602 Action [[1.12214097]] Reward -5.418199743700058 Loss 0.5134015083312988\n",
      "Episode 13 Step 2603 Action [[1.00592106]] Reward -6.054229150761282 Loss 0.8478341102600098\n",
      "Episode 13 Step 2604 Action [[1.0831005]] Reward -6.820301846051724 Loss 0.1466919183731079\n",
      "Episode 13 Step 2605 Action [[1.04431566]] Reward -7.6747665245387795 Loss 0.8890604972839355\n",
      "Episode 13 Step 2606 Action [[1.3226375]] Reward -8.5947330743721 Loss 0.5879458785057068\n",
      "Episode 13 Step 2607 Action [[1.25328042]] Reward -9.498370297062896 Loss 1.1675947904586792\n",
      "Episode 13 Step 2608 Action [[1.14165777]] Reward -10.379498373710948 Loss 0.3508467674255371\n",
      "Episode 13 Step 2609 Action [[1.04330104]] Reward -10.190953241169211 Loss 0.9737269282341003\n",
      "Episode 13 Step 2610 Action [[1.06019854]] Reward -9.276880175101583 Loss 0.2925100326538086\n",
      "Episode 13 Step 2611 Action [[0.92501246]] Reward -8.448198724328448 Loss 0.2211136668920517\n",
      "Episode 13 Step 2612 Action [[0.92567005]] Reward -7.745199099674287 Loss 0.6786096096038818\n",
      "Episode 13 Step 2613 Action [[0.79278658]] Reward -7.187582612607423 Loss 0.2841178774833679\n",
      "Episode 13 Step 2614 Action [[0.55967921]] Reward -6.799152148441665 Loss 0.23069694638252258\n",
      "Episode 13 Step 2615 Action [[0.42015628]] Reward -6.5844839425907 Loss 0.17549054324626923\n",
      "Episode 13 Step 2616 Action [[0.65349763]] Reward -6.538405524977362 Loss 0.15775331854820251\n",
      "Episode 13 Step 2617 Action [[0.85350047]] Reward -6.6720599021483 Loss 0.08913400769233704\n",
      "Episode 13 Step 2618 Action [[1.21196833]] Reward -7.000706861405472 Loss 0.1272542029619217\n",
      "Episode 13 Step 2619 Action [[1.60345325]] Reward -7.545078240695074 Loss 0.21676479279994965\n",
      "Episode 13 Step 2620 Action [[1.81112637]] Reward -8.320600088114656 Loss 0.18152379989624023\n",
      "Episode 13 Step 2621 Action [[1.80601402]] Reward -9.30919806473304 Loss 0.1050885021686554\n",
      "Episode 13 Step 2622 Action [[1.61113021]] Reward -10.456467049663452 Loss 1.1039994955062866\n",
      "Episode 13 Step 2623 Action [[1.86526345]] Reward -10.270117531320018 Loss 0.22123493254184723\n",
      "Episode 13 Step 2624 Action [[2.07501313]] Reward -9.374655911349803 Loss 0.9912549257278442\n",
      "Episode 13 Step 2625 Action [[1.68926363]] Reward -8.444190694812301 Loss 0.433383584022522\n",
      "Episode 13 Step 2626 Action [[1.35043292]] Reward -7.477613953987184 Loss 0.3299909830093384\n",
      "Episode 13 Step 2627 Action [[1.03119675]] Reward -6.523113110205655 Loss 0.14267289638519287\n",
      "Episode 13 Step 2628 Action [[0.21008964]] Reward -5.635801952467927 Loss 3.146303176879883\n",
      "Episode 13 Step 2629 Action [[-0.17975366]] Reward -4.847982922138297 Loss 0.07423773407936096\n",
      "Episode 13 Step 2630 Action [[0.1476288]] Reward -4.253590387317235 Loss 1.0874367952346802\n",
      "Episode 13 Step 2631 Action [[0.44662293]] Reward -3.897251168619782 Loss 0.04779132828116417\n",
      "Episode 13 Step 2632 Action [[0.80022615]] Reward -3.755562136923067 Loss 0.2296338677406311\n",
      "Episode 13 Step 2633 Action [[0.93882708]] Reward -3.802938200885458 Loss 0.08512367308139801\n",
      "Episode 13 Step 2634 Action [[0.94106022]] Reward -4.0237296486463325 Loss 5.465071201324463\n",
      "Episode 13 Step 2635 Action [[1.07771083]] Reward -4.417917807464315 Loss 0.14606529474258423\n",
      "Episode 13 Step 2636 Action [[0.94105873]] Reward -4.971108180719324 Loss 0.11907735466957092\n",
      "Episode 13 Step 2637 Action [[0.88196164]] Reward -5.6947554481651625 Loss 0.06923559308052063\n",
      "Episode 13 Step 2638 Action [[1.19064108]] Reward -6.575658386270252 Loss 0.1110551655292511\n",
      "Episode 13 Step 2639 Action [[1.01431809]] Reward -7.54101626978701 Loss 0.06126407906413078\n",
      "Episode 13 Step 2640 Action [[0.65736971]] Reward -8.601024611578454 Loss 0.08055788278579712\n",
      "Episode 13 Step 2641 Action [[0.71773985]] Reward -9.748539399119077 Loss 0.0918218344449997\n",
      "Episode 13 Step 2642 Action [[0.75677117]] Reward -10.886876581438326 Loss 0.07046616822481155\n",
      "Episode 13 Step 2643 Action [[0.36227705]] Reward -10.345420244089444 Loss 0.08153155446052551\n",
      "Episode 13 Step 2644 Action [[-0.4711567]] Reward -9.24186409356162 Loss 0.24511182308197021\n",
      "Episode 13 Step 2645 Action [[-1.2245535]] Reward -8.24259763311041 Loss 0.04929248243570328\n",
      "Episode 13 Step 2646 Action [[-0.8442158]] Reward -7.336746836791686 Loss 0.03363170474767685\n",
      "Episode 13 Step 2647 Action [[-0.76074886]] Reward -6.467179815802124 Loss 0.09627213329076767\n",
      "Episode 13 Step 2648 Action [[-0.62459689]] Reward -5.700416910326223 Loss 0.13204360008239746\n",
      "Episode 13 Step 2649 Action [[-0.82974773]] Reward -5.067310981395755 Loss 0.06359831243753433\n",
      "Episode 13 Step 2650 Action [[-0.78572205]] Reward -4.598742467302156 Loss 0.08268824219703674\n",
      "Episode 13 Step 2651 Action [[-0.77287762]] Reward -4.291365598270839 Loss 0.04689302295446396\n",
      "Episode 13 Step 2652 Action [[-0.67530391]] Reward -4.153854280901011 Loss 0.041782766580581665\n",
      "Episode 13 Step 2653 Action [[-0.62956165]] Reward -4.193838168996509 Loss 0.09591387212276459\n",
      "Episode 13 Step 2654 Action [[-0.54114729]] Reward -4.417925625694684 Loss 0.05994526296854019\n",
      "Episode 13 Step 2655 Action [[-0.8709186]] Reward -4.833308800935254 Loss 0.06425341218709946\n",
      "Episode 13 Step 2656 Action [[-0.78537756]] Reward -5.404350134507483 Loss 0.038494355976581573\n",
      "Episode 13 Step 2657 Action [[-0.44967117]] Reward -6.1379601944436075 Loss 0.10726908594369888\n",
      "Episode 13 Step 2658 Action [[0.22457558]] Reward -7.052132123884804 Loss 0.06977272778749466\n",
      "Episode 13 Step 2659 Action [[0.44145313]] Reward -8.189051261068483 Loss 0.06074755638837814\n",
      "Episode 13 Step 2660 Action [[0.60278425]] Reward -9.488336791348246 Loss 0.04382210224866867\n",
      "Episode 13 Step 2661 Action [[0.6878977]] Reward -10.897580003238941 Loss 5.2183990478515625\n",
      "Episode 13 Step 2662 Action [[0.62551631]] Reward -10.871960246192748 Loss 0.24080786108970642\n",
      "Episode 13 Step 2663 Action [[0.8723666]] Reward -9.662419860044457 Loss 1.1816717386245728\n",
      "Episode 13 Step 2664 Action [[0.79193158]] Reward -8.472939947110987 Loss 0.32156139612197876\n",
      "Episode 13 Step 2665 Action [[0.72609182]] Reward -7.31443721033061 Loss 0.22253412008285522\n",
      "Episode 13 Step 2666 Action [[0.50031953]] Reward -6.240401536545087 Loss 0.20739208161830902\n",
      "Episode 13 Step 2667 Action [[-0.0387154]] Reward -5.287614823691936 Loss 0.15944431722164154\n",
      "Episode 13 Step 2668 Action [[0.20819228]] Reward -4.490746880556463 Loss 3.442505121231079\n",
      "Episode 13 Step 2669 Action [[0.37130527]] Reward -3.926971009767089 Loss 0.3038017749786377\n",
      "Episode 13 Step 2670 Action [[0.58992374]] Reward -3.5818968122006383 Loss 0.5412169694900513\n",
      "Episode 13 Step 2671 Action [[1.22973619]] Reward -3.4435772179837576 Loss 0.41186487674713135\n",
      "Episode 13 Step 2672 Action [[1.48098554]] Reward -3.478602300503252 Loss 0.1022743433713913\n",
      "Episode 13 Step 2673 Action [[1.63104534]] Reward -3.658820375385602 Loss 0.1987808644771576\n",
      "Episode 13 Step 2674 Action [[1.88194036]] Reward -3.9744376640873984 Loss 0.30500108003616333\n",
      "Episode 13 Step 2675 Action [[1.89695523]] Reward -4.405877464162878 Loss 0.1805049479007721\n",
      "Episode 13 Step 2676 Action [[1.52045929]] Reward -4.953837253524995 Loss 4.058882713317871\n",
      "Episode 13 Step 2677 Action [[1.50764345]] Reward -5.65269739690943 Loss 3.031766891479492\n",
      "Episode 13 Step 2678 Action [[1.06611667]] Reward -6.468531751061309 Loss 1.4040625095367432\n",
      "Episode 13 Step 2679 Action [[1.35518539]] Reward -7.4356119800136815 Loss 0.5675354599952698\n",
      "Episode 13 Step 2680 Action [[1.16951461]] Reward -8.442517176990822 Loss 0.1433568000793457\n",
      "Episode 13 Step 2681 Action [[1.12371657]] Reward -9.497778845584499 Loss 5.558725833892822\n",
      "Episode 13 Step 2682 Action [[0.89555796]] Reward -10.542153381896924 Loss 0.943924069404602\n",
      "Episode 13 Step 2683 Action [[0.90024258]] Reward -10.498971992542995 Loss 0.1721697598695755\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 13 Step 2684 Action [[0.65104369]] Reward -9.41647419289854 Loss 1.1434476375579834\n",
      "Episode 13 Step 2685 Action [[0.73162786]] Reward -8.422236226020225 Loss 0.9280492067337036\n",
      "Episode 13 Step 2686 Action [[0.04083564]] Reward -7.528944572936752 Loss 2.901689052581787\n",
      "Episode 13 Step 2687 Action [[-0.65870458]] Reward -6.801973845428183 Loss 0.44212549924850464\n",
      "Episode 13 Step 2688 Action [[-0.83843619]] Reward -6.221509451397892 Loss 0.3353782594203949\n",
      "Episode 13 Step 2689 Action [[-1.0164754]] Reward -5.758507111063063 Loss 0.3710344135761261\n",
      "Episode 13 Step 2690 Action [[-0.69175671]] Reward -5.415636841387222 Loss 0.1381281316280365\n",
      "Episode 13 Step 2691 Action [[-0.83167185]] Reward -5.200608912404989 Loss 0.38771411776542664\n",
      "Episode 13 Step 2692 Action [[-0.70975065]] Reward -5.129460777295417 Loss 0.2412809431552887\n",
      "Episode 13 Step 2693 Action [[-0.44972822]] Reward -5.206502886976474 Loss 0.1995488703250885\n",
      "Episode 13 Step 2694 Action [[-0.35926524]] Reward -5.449334360839177 Loss 0.20039454102516174\n",
      "Episode 13 Step 2695 Action [[0.26332779]] Reward -5.861606689049681 Loss 0.25963613390922546\n",
      "Episode 13 Step 2696 Action [[0.45727566]] Reward -6.489770351807769 Loss 0.09542910754680634\n",
      "Episode 13 Step 2697 Action [[1.40353931]] Reward -7.32386208401975 Loss 0.07535593956708908\n",
      "Episode 13 Step 2698 Action [[1.63979368]] Reward -8.445509598072622 Loss 0.13745935261249542\n",
      "Episode 13 Step 2699 Action [[1.62150872]] Reward -9.792200014632355 Loss 0.19681864976882935\n",
      "Episode 13 Step 2700 Action [[1.46226523]] Reward -11.280920593436383 Loss 0.2746363580226898\n",
      "Episode 13 Step 2701 Action [[1.43351548]] Reward -10.466824539706645 Loss 1.9088375568389893\n",
      "Episode 13 Step 2702 Action [[1.341466]] Reward -9.322425784054788 Loss 0.09729929268360138\n",
      "Episode 13 Step 2703 Action [[1.26270442]] Reward -8.1515882437051 Loss 0.08392880856990814\n",
      "Episode 13 Step 2704 Action [[0.74361893]] Reward -7.005227380605855 Loss 1.392174482345581\n",
      "Episode 13 Step 2705 Action [[0.72477273]] Reward -5.904171486275733 Loss 0.0925007313489914\n",
      "Episode 13 Step 2706 Action [[0.73878904]] Reward -4.95183097180904 Loss 0.24279536306858063\n",
      "Episode 13 Step 2707 Action [[0.55699035]] Reward -4.17651481251205 Loss 0.24288557469844818\n",
      "Episode 13 Step 2708 Action [[0.53236479]] Reward -3.584570550248993 Loss 0.08586830645799637\n",
      "Episode 13 Step 2709 Action [[1.13881369]] Reward -3.1956790938610236 Loss 0.11622598767280579\n",
      "Episode 13 Step 2710 Action [[1.25580636]] Reward -3.0022008719963953 Loss 0.23581676185131073\n",
      "Episode 13 Step 2711 Action [[1.54110184]] Reward -2.969616950489861 Loss 0.16165383160114288\n",
      "Episode 13 Step 2712 Action [[1.64633247]] Reward -3.0817762432619817 Loss 0.07315273582935333\n",
      "Episode 13 Step 2713 Action [[1.5953796]] Reward -3.330677356794653 Loss 0.09514965116977692\n",
      "Episode 13 Step 2714 Action [[1.48401569]] Reward -3.7240461195676726 Loss 0.15559762716293335\n",
      "Episode 13 Step 2715 Action [[1.40310693]] Reward -4.274402984748565 Loss 0.06082594767212868\n",
      "Episode 13 Step 2716 Action [[1.55563449]] Reward -4.987207222570701 Loss 0.09860768914222717\n",
      "Episode 13 Step 2717 Action [[1.05878383]] Reward -5.830301709352774 Loss 0.17261546850204468\n",
      "Episode 13 Step 2718 Action [[1.11193045]] Reward -6.86328080406457 Loss 1.1401551961898804\n",
      "Episode 13 Step 2719 Action [[1.45881005]] Reward -8.007967378055938 Loss 3.2281908988952637\n",
      "Episode 13 Step 2720 Action [[1.00442435]] Reward -9.163787049107869 Loss 0.5094349980354309\n",
      "Episode 13 Step 2721 Action [[0.75488088]] Reward -10.387077898156896 Loss 0.4586195945739746\n",
      "Episode 13 Step 2722 Action [[0.72048227]] Reward -11.255458686244996 Loss 0.13141882419586182\n",
      "Episode 13 Step 2723 Action [[0.4765029]] Reward -10.000794995889644 Loss 0.2096863090991974\n",
      "Episode 13 Step 2724 Action [[-0.16267785]] Reward -8.81010510740097 Loss 0.27757734060287476\n",
      "Episode 13 Step 2725 Action [[-0.92258647]] Reward -7.742997095941117 Loss 0.18241733312606812\n",
      "Episode 13 Step 2726 Action [[-1.02977243]] Reward -6.810594007169943 Loss 0.04485968500375748\n",
      "Episode 13 Step 2727 Action [[-0.50619422]] Reward -5.973286826964043 Loss 0.21620309352874756\n",
      "Episode 13 Step 2728 Action [[-0.60852159]] Reward -5.233350501277183 Loss 3.5444812774658203\n",
      "Episode 13 Step 2729 Action [[-0.66757159]] Reward -4.662944096837296 Loss 0.15389719605445862\n",
      "Episode 13 Step 2730 Action [[-0.52781269]] Reward -4.2673651586464825 Loss 0.08533422648906708\n",
      "Episode 13 Step 2731 Action [[-0.74627616]] Reward -4.052559366987858 Loss 0.03839181363582611\n",
      "Episode 13 Step 2732 Action [[-0.74487863]] Reward -4.0222612240572495 Loss 0.2584344148635864\n",
      "Episode 13 Step 2733 Action [[-0.82511732]] Reward -4.171569165298266 Loss 0.12797735631465912\n",
      "Episode 13 Step 2734 Action [[-0.71627995]] Reward -4.495656688312679 Loss 3.028656482696533\n",
      "Episode 13 Step 2735 Action [[-0.70853811]] Reward -5.002313369989844 Loss 0.47551560401916504\n",
      "Episode 13 Step 2736 Action [[-0.80282162]] Reward -5.684136521539545 Loss 0.5190827250480652\n",
      "Episode 13 Step 2737 Action [[-0.41796428]] Reward -6.514904710027788 Loss 0.10230881720781326\n",
      "Episode 13 Step 2738 Action [[0.7885127]] Reward -7.525707015349797 Loss 0.11423555761575699\n",
      "Episode 13 Step 2739 Action [[1.34384452]] Reward -8.829544103909676 Loss 0.33162787556648254\n",
      "Episode 13 Step 2740 Action [[1.5893628]] Reward -10.367607111627816 Loss 0.42209941148757935\n",
      "Episode 13 Step 2741 Action [[1.6964004]] Reward -11.664923575685773 Loss 0.07136421650648117\n",
      "Episode 13 Step 2742 Action [[1.40567447]] Reward -10.471622934159411 Loss 0.2369610220193863\n",
      "Episode 13 Step 2743 Action [[1.00344513]] Reward -9.186254398072858 Loss 0.25872480869293213\n",
      "Episode 13 Step 2744 Action [[1.21909473]] Reward -7.857635140166224 Loss 0.13194650411605835\n",
      "Episode 13 Step 2745 Action [[1.27556217]] Reward -6.618690097681478 Loss 0.06526431441307068\n",
      "Episode 13 Step 2746 Action [[1.66148062]] Reward -5.496542220620951 Loss 0.13719919323921204\n",
      "Episode 13 Step 2747 Action [[1.24345921]] Reward -4.545567458687783 Loss 0.2237459123134613\n",
      "Episode 13 Step 2748 Action [[0.73596734]] Reward -3.716357868448245 Loss 0.09126448631286621\n",
      "Episode 13 Step 2749 Action [[0.15444474]] Reward -3.0407641679605515 Loss 0.12413837760686874\n",
      "Episode 13 Step 2750 Action [[0.03318388]] Reward -2.5509073434608376 Loss 0.10160136967897415\n",
      "Episode 13 Step 2751 Action [[0.50613151]] Reward -2.284699339367677 Loss 0.1965809166431427\n",
      "Episode 13 Step 2752 Action [[0.79811242]] Reward -2.2310628734360547 Loss 0.06284341216087341\n",
      "Episode 13 Step 2753 Action [[0.95517753]] Reward -2.3568731871705415 Loss 0.06236300244927406\n",
      "Episode 13 Step 2754 Action [[0.84827937]] Reward -2.6525195997428286 Loss 0.0715952068567276\n",
      "Episode 13 Step 2755 Action [[1.00819896]] Reward -3.1387299965580078 Loss 3.5450456142425537\n",
      "Episode 13 Step 2756 Action [[0.60574252]] Reward -3.8104972602686598 Loss 3.2603671550750732\n",
      "Episode 13 Step 2757 Action [[0.83836222]] Reward -4.73467390081004 Loss 0.671219527721405\n",
      "Episode 13 Step 2758 Action [[1.0017486]] Reward -5.857982860496945 Loss 0.21943792700767517\n",
      "Episode 13 Step 2759 Action [[0.79568857]] Reward -7.153148851346242 Loss 0.13797883689403534\n",
      "Episode 13 Step 2760 Action [[0.71173908]] Reward -8.629632410890872 Loss 0.16387939453125\n",
      "Episode 13 Step 2761 Action [[0.69504766]] Reward -10.213774348909702 Loss 0.20080646872520447\n",
      "Episode 13 Step 2762 Action [[0.6165937]] Reward -11.815104166065836 Loss 0.17362068593502045\n",
      "Episode 13 Step 2763 Action [[0.7513853]] Reward -11.344371190961558 Loss 0.9823409914970398\n",
      "Episode 13 Step 2764 Action [[0.17756336]] Reward -9.765737317938243 Loss 0.29441699385643005\n",
      "Episode 13 Step 2765 Action [[-1.29713257]] Reward -8.316764470955533 Loss 0.31390994787216187\n",
      "Episode 13 Step 2766 Action [[-1.26445393]] Reward -7.10223690791229 Loss 6.356422424316406\n",
      "Episode 13 Step 2767 Action [[-1.08628351]] Reward -5.967876995433912 Loss 0.18330055475234985\n",
      "Episode 13 Step 2768 Action [[-1.38528931]] Reward -4.9491524105841 Loss 5.904462814331055\n",
      "Episode 13 Step 2769 Action [[-1.0368095]] Reward -4.110470688965595 Loss 0.29640114307403564\n",
      "Episode 13 Step 2770 Action [[-1.21753193]] Reward -3.417833570672 Loss 0.21720877289772034\n",
      "Episode 13 Step 2771 Action [[-1.67672569]] Reward -2.911111572058898 Loss 0.27278780937194824\n",
      "Episode 13 Step 2772 Action [[-1.78002402]] Reward -2.572800750426854 Loss 1.79679536819458\n",
      "Episode 13 Step 2773 Action [[-1.8493297]] Reward -2.3662570519092125 Loss 0.4295041561126709\n",
      "Episode 13 Step 2774 Action [[-1.58764623]] Reward -2.278960894277602 Loss 3.958019733428955\n",
      "Episode 13 Step 2775 Action [[-1.59856571]] Reward -2.31606575301614 Loss 0.2155531346797943\n",
      "Episode 13 Step 2776 Action [[-1.56670821]] Reward -2.483891425743658 Loss 0.44611847400665283\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 13 Step 2777 Action [[-1.12058493]] Reward -2.789587792368246 Loss 0.1581653654575348\n",
      "Episode 13 Step 2778 Action [[-1.02625547]] Reward -3.2801722225783103 Loss 0.1209753081202507\n",
      "Episode 13 Step 2779 Action [[-1.22048579]] Reward -3.9668839036960963 Loss 0.36770182847976685\n",
      "Episode 13 Step 2780 Action [[-1.14190301]] Reward -4.830011002864722 Loss 0.3625237047672272\n",
      "Episode 13 Step 2781 Action [[-1.07403055]] Reward -5.888977700694489 Loss 5.890777587890625\n",
      "Episode 13 Step 2782 Action [[-0.7814797]] Reward -7.128326389266909 Loss 0.36132895946502686\n",
      "Episode 13 Step 2783 Action [[0.62146482]] Reward -8.550850446717881 Loss 1.3185958862304688\n",
      "Episode 13 Step 2784 Action [[1.00875379]] Reward -10.322607100558526 Loss 1.3117291927337646\n",
      "Episode 13 Step 2785 Action [[1.30320121]] Reward -12.268314102348269 Loss 0.2801763415336609\n",
      "Episode 13 Step 2786 Action [[1.23353242]] Reward -11.583128814097398 Loss 0.9755114316940308\n",
      "Episode 13 Step 2787 Action [[1.33169895]] Reward -10.071023625486918 Loss 1.060584306716919\n",
      "Episode 13 Step 2788 Action [[1.18912711]] Reward -8.552383156706139 Loss 0.23949486017227173\n",
      "Episode 13 Step 2789 Action [[0.88994147]] Reward -7.065671743693532 Loss 0.37300586700439453\n",
      "Episode 13 Step 2790 Action [[0.70583522]] Reward -5.672629461702774 Loss 0.589211642742157\n",
      "Episode 13 Step 2791 Action [[-0.24664701]] Reward -4.454388543300817 Loss 0.673829972743988\n",
      "Episode 13 Step 2792 Action [[-0.79269346]] Reward -3.3918072153940813 Loss 0.14036361873149872\n",
      "Episode 13 Step 2793 Action [[-0.71726992]] Reward -2.581383760669155 Loss 0.1757749319076538\n",
      "Episode 13 Step 2794 Action [[-0.5006556]] Reward -2.067043557035847 Loss 5.232077121734619\n",
      "Episode 13 Step 2795 Action [[-0.03978782]] Reward -1.8189406327847248 Loss 0.08821484446525574\n",
      "Episode 13 Step 2796 Action [[0.35149318]] Reward -1.7989601048684996 Loss 0.6983224153518677\n",
      "Episode 13 Step 2797 Action [[0.87757282]] Reward -1.9705161548994985 Loss 0.36300212144851685\n",
      "Episode 13 Step 2798 Action [[0.79553605]] Reward -2.3021878043533923 Loss 0.4536711871623993\n",
      "Episode 13 Step 2799 Action [[0.78664495]] Reward -2.8213167929456966 Loss 1.656195878982544\n",
      "TOTAL REWARD @ 13-th Episode  : Reward -1267.734805688691\n",
      "Total Step: 2800\n",
      "\n",
      "Episode : 14 Replay Buffer 2800\n",
      "[-0.118536    0.99294976  0.82421152]\n",
      "Episode 14 Step 2800 Action [[-1.283878]] Reward -2.9243684525204374 Loss 0.14286956191062927\n",
      "Episode 14 Step 2801 Action [[-1.39656529]] Reward -3.2834539778530605 Loss 0.22073134779930115\n",
      "Episode 14 Step 2802 Action [[-1.27845985]] Reward -3.7999224080873444 Loss 0.1876106560230255\n",
      "Episode 14 Step 2803 Action [[-1.26492458]] Reward -4.494619397191149 Loss 0.2141396701335907\n",
      "Episode 14 Step 2804 Action [[-1.53598207]] Reward -5.3640521259344895 Loss 0.2830940783023834\n",
      "Episode 14 Step 2805 Action [[-1.2660163]] Reward -6.356268651703546 Loss 0.1713714450597763\n",
      "Episode 14 Step 2806 Action [[-0.11996802]] Reward -7.504125328959119 Loss 0.07112756371498108\n",
      "Episode 14 Step 2807 Action [[0.43013865]] Reward -8.928645320714994 Loss 0.11182668805122375\n",
      "Episode 14 Step 2808 Action [[0.14585621]] Reward -10.549169257247172 Loss 0.24287700653076172\n",
      "Episode 14 Step 2809 Action [[0.52855548]] Reward -12.08648902243463 Loss 2.225062131881714\n",
      "Episode 14 Step 2810 Action [[0.73915355]] Reward -10.70321315195288 Loss 0.07755309343338013\n",
      "Episode 14 Step 2811 Action [[0.88812488]] Reward -9.314508803343882 Loss 0.482854425907135\n",
      "Episode 14 Step 2812 Action [[1.22267614]] Reward -7.9604506861611855 Loss 0.272291898727417\n",
      "Episode 14 Step 2813 Action [[1.17858285]] Reward -6.7087201467364475 Loss 0.13468486070632935\n",
      "Episode 14 Step 2814 Action [[1.48059213]] Reward -5.5624277180721 Loss 0.305850088596344\n",
      "Episode 14 Step 2815 Action [[1.30325261]] Reward -4.586409801165046 Loss 0.2926419973373413\n",
      "Episode 14 Step 2816 Action [[1.05847627]] Reward -3.755039215842978 Loss 0.09665950387716293\n",
      "Episode 14 Step 2817 Action [[0.35079213]] Reward -3.083742888021751 Loss 3.853834390640259\n",
      "Episode 14 Step 2818 Action [[-0.05825418]] Reward -2.5752594039955112 Loss 0.5170233845710754\n",
      "Episode 14 Step 2819 Action [[0.31398899]] Reward -2.2774638177536763 Loss 0.704761266708374\n",
      "Episode 14 Step 2820 Action [[1.0849644]] Reward -2.202150518142198 Loss 3.7099103927612305\n",
      "Episode 14 Step 2821 Action [[1.50703661]] Reward -2.2995482099113853 Loss 0.09705372154712677\n",
      "Episode 14 Step 2822 Action [[1.3737048]] Reward -2.532782567842603 Loss 6.494802951812744\n",
      "Episode 14 Step 2823 Action [[1.39248208]] Reward -2.921038445942994 Loss 0.4025317430496216\n",
      "Episode 14 Step 2824 Action [[1.0049457]] Reward -3.4709519933084287 Loss 0.4912077784538269\n",
      "Episode 14 Step 2825 Action [[1.07839187]] Reward -4.237605250482355 Loss 0.30089622735977173\n",
      "Episode 14 Step 2826 Action [[1.2416969]] Reward -5.199231406625556 Loss 5.77925968170166\n",
      "Episode 14 Step 2827 Action [[1.09950562]] Reward -6.328193485975398 Loss 2.1893765926361084\n",
      "Episode 14 Step 2828 Action [[0.92303533]] Reward -7.633528000299732 Loss 0.5139333009719849\n",
      "Episode 14 Step 2829 Action [[1.07581324]] Reward -9.082948977679367 Loss 0.2480844408273697\n",
      "Episode 14 Step 2830 Action [[1.28537889]] Reward -10.558252281394605 Loss 0.24280977249145508\n",
      "Episode 14 Step 2831 Action [[1.51674508]] Reward -11.962151175249787 Loss 0.2338077276945114\n",
      "Episode 14 Step 2832 Action [[1.45785245]] Reward -10.614416464895735 Loss 0.13191603124141693\n",
      "Episode 14 Step 2833 Action [[1.9837492]] Reward -9.136138391697914 Loss 0.07175789028406143\n",
      "Episode 14 Step 2834 Action [[1.50212389]] Reward -7.755311817823142 Loss 0.08357804268598557\n",
      "Episode 14 Step 2835 Action [[0.12921389]] Reward -6.635388368900549 Loss 0.08097265660762787\n",
      "Episode 14 Step 2836 Action [[-0.57130085]] Reward -5.814727789123775 Loss 0.21768862009048462\n",
      "Episode 14 Step 2837 Action [[-0.69239632]] Reward -5.195441555091763 Loss 3.8837215900421143\n",
      "Episode 14 Step 2838 Action [[-0.71468569]] Reward -4.73812703380299 Loss 0.15143975615501404\n",
      "Episode 14 Step 2839 Action [[-0.61349929]] Reward -4.445556555924899 Loss 0.07221594452857971\n",
      "Episode 14 Step 2840 Action [[-0.65649715]] Reward -4.325676789116361 Loss 0.0461733415722847\n",
      "Episode 14 Step 2841 Action [[-0.4264169]] Reward -4.383957278040872 Loss 0.0929320827126503\n",
      "Episode 14 Step 2842 Action [[-0.49436283]] Reward -4.634394182251946 Loss 0.14901034533977509\n",
      "Episode 14 Step 2843 Action [[-0.35700356]] Reward -5.072113447096947 Loss 0.0968446284532547\n",
      "Episode 14 Step 2844 Action [[-0.12324248]] Reward -5.704842897670495 Loss 0.06649688631296158\n",
      "Episode 14 Step 2845 Action [[0.41651182]] Reward -6.542000794863931 Loss 6.185434341430664\n",
      "Episode 14 Step 2846 Action [[1.33913264]] Reward -7.620243671824339 Loss 0.24323205649852753\n",
      "Episode 14 Step 2847 Action [[1.55941235]] Reward -9.003234001123419 Loss 0.22378718852996826\n",
      "Episode 14 Step 2848 Action [[1.51835076]] Reward -10.599017988516179 Loss 0.42929786443710327\n",
      "Episode 14 Step 2849 Action [[1.59349731]] Reward -11.54075477175278 Loss 0.42337560653686523\n",
      "Episode 14 Step 2850 Action [[1.7079539]] Reward -10.32252222479257 Loss 0.1323087215423584\n",
      "Episode 14 Step 2851 Action [[1.52084818]] Reward -9.061120216798802 Loss 0.07424190640449524\n",
      "Episode 14 Step 2852 Action [[1.67266762]] Reward -7.771369885103964 Loss 0.09653176367282867\n",
      "Episode 14 Step 2853 Action [[1.15052162]] Reward -6.547534647735872 Loss 0.12521657347679138\n",
      "Episode 14 Step 2854 Action [[0.88451981]] Reward -5.375874457758448 Loss 0.2638736069202423\n",
      "Episode 14 Step 2855 Action [[0.09614727]] Reward -4.346436213319147 Loss 0.19593489170074463\n",
      "Episode 14 Step 2856 Action [[-0.6118478]] Reward -3.4701000803163904 Loss 0.10572142899036407\n",
      "Episode 14 Step 2857 Action [[-0.35404658]] Reward -2.815707506768297 Loss 0.05749290809035301\n",
      "Episode 14 Step 2858 Action [[0.17200306]] Reward -2.4457612713943715 Loss 1.2649697065353394\n",
      "Episode 14 Step 2859 Action [[0.94957102]] Reward -2.322440317193318 Loss 0.0655977874994278\n",
      "Episode 14 Step 2860 Action [[1.28980792]] Reward -2.386633162068697 Loss 0.10765667259693146\n",
      "Episode 14 Step 2861 Action [[1.5221713]] Reward -2.6028733160549837 Loss 0.08879663050174713\n",
      "Episode 14 Step 2862 Action [[1.65447203]] Reward -2.9607008610631005 Loss 0.15784724056720734\n",
      "Episode 14 Step 2863 Action [[1.79564172]] Reward -3.4615418745254685 Loss 0.18910539150238037\n",
      "Episode 14 Step 2864 Action [[1.69004695]] Reward -4.102344584529967 Loss 5.793047904968262\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 14 Step 2865 Action [[1.58467428]] Reward -4.905258322480217 Loss 0.09383540600538254\n",
      "Episode 14 Step 2866 Action [[2.08961593]] Reward -5.873340924231235 Loss 0.07874469459056854\n",
      "Episode 14 Step 2867 Action [[2.01127431]] Reward -6.914675125017806 Loss 0.0412420816719532\n",
      "Episode 14 Step 2868 Action [[1.79340648]] Reward -8.029489746226176 Loss 0.048201605677604675\n",
      "Episode 14 Step 2869 Action [[1.82018761]] Reward -9.203602491419657 Loss 0.07866024971008301\n",
      "Episode 14 Step 2870 Action [[1.4464081]] Reward -10.351556583907106 Loss 0.07639247179031372\n",
      "Episode 14 Step 2871 Action [[0.82854842]] Reward -11.328786187606617 Loss 0.020240068435668945\n",
      "Episode 14 Step 2872 Action [[0.99696641]] Reward -10.077319254782955 Loss 0.039661705493927\n",
      "Episode 14 Step 2873 Action [[1.12693538]] Reward -8.865091420069836 Loss 0.036099933087825775\n",
      "Episode 14 Step 2874 Action [[0.1540335]] Reward -7.755653722585142 Loss 0.09004321694374084\n",
      "Episode 14 Step 2875 Action [[-0.88066976]] Reward -6.855833095938174 Loss 0.05103661119937897\n",
      "Episode 14 Step 2876 Action [[-1.00139408]] Reward -6.133126987229039 Loss 0.04154020547866821\n",
      "Episode 14 Step 2877 Action [[-1.43433284]] Reward -5.52570741803122 Loss 4.172112941741943\n",
      "Episode 14 Step 2878 Action [[-1.43104077]] Reward -5.04740633730208 Loss 0.03772120177745819\n",
      "Episode 14 Step 2879 Action [[-1.55260946]] Reward -4.682959017114161 Loss 0.2550504803657532\n",
      "Episode 14 Step 2880 Action [[-1.59963079]] Reward -4.439455794609823 Loss 0.11618714034557343\n",
      "Episode 14 Step 2881 Action [[-1.68669041]] Reward -4.31563141591455 Loss 1.3671743869781494\n",
      "Episode 14 Step 2882 Action [[-1.45289197]] Reward -4.308601233311446 Loss 0.06097333878278732\n",
      "Episode 14 Step 2883 Action [[-1.48505452]] Reward -4.42981871561099 Loss 0.07993848621845245\n",
      "Episode 14 Step 2884 Action [[-1.44883024]] Reward -4.678658971790591 Loss 0.0990896001458168\n",
      "Episode 14 Step 2885 Action [[-1.41467546]] Reward -5.055266728700998 Loss 0.2046651989221573\n",
      "Episode 14 Step 2886 Action [[-1.02795962]] Reward -5.554992123479214 Loss 0.08063767850399017\n",
      "Episode 14 Step 2887 Action [[-0.40782078]] Reward -6.204631393345246 Loss 0.096010223031044\n",
      "Episode 14 Step 2888 Action [[0.42255761]] Reward -7.041876310324238 Loss 0.12448632717132568\n",
      "Episode 14 Step 2889 Action [[0.41934419]] Reward -8.11385043753607 Loss 0.042982280254364014\n",
      "Episode 14 Step 2890 Action [[0.8276503]] Reward -9.325783220484338 Loss 0.043748639523983\n",
      "Episode 14 Step 2891 Action [[1.03830492]] Reward -10.6861008761724 Loss 0.06586266309022903\n",
      "Episode 14 Step 2892 Action [[0.99107348]] Reward -10.858946201299629 Loss 0.08626532554626465\n",
      "Episode 14 Step 2893 Action [[1.08088939]] Reward -9.716939475367612 Loss 0.07532553374767303\n",
      "Episode 14 Step 2894 Action [[0.91722236]] Reward -8.564057283748545 Loss 0.020618973299860954\n",
      "Episode 14 Step 2895 Action [[1.08654308]] Reward -7.422378364360839 Loss 0.03324658423662186\n",
      "Episode 14 Step 2896 Action [[0.86563674]] Reward -6.369550126333952 Loss 0.11105460673570633\n",
      "Episode 14 Step 2897 Action [[0.41333329]] Reward -5.413147527901098 Loss 0.028185561299324036\n",
      "Episode 14 Step 2898 Action [[0.57562518]] Reward -4.589653833035148 Loss 2.694981813430786\n",
      "Episode 14 Step 2899 Action [[0.61327085]] Reward -3.968768073039514 Loss 0.030920278280973434\n",
      "Episode 14 Step 2900 Action [[0.32149788]] Reward -3.5437019621019314 Loss 0.02806016616523266\n",
      "Episode 14 Step 2901 Action [[0.72856716]] Reward -3.3174358435496023 Loss 0.042442332953214645\n",
      "Episode 14 Step 2902 Action [[0.80098399]] Reward -3.2940131031363737 Loss 0.06417647004127502\n",
      "Episode 14 Step 2903 Action [[0.8667294]] Reward -3.4570675260843493 Loss 0.04017753154039383\n",
      "Episode 14 Step 2904 Action [[0.86181118]] Reward -3.804025465902911 Loss 0.07630770653486252\n",
      "Episode 14 Step 2905 Action [[0.96796488]] Reward -4.338845051184479 Loss 0.052939523011446\n",
      "Episode 14 Step 2906 Action [[1.12619589]] Reward -5.05042027890103 Loss 4.051397323608398\n",
      "Episode 14 Step 2907 Action [[1.08197381]] Reward -5.915503833101582 Loss 1.6722190380096436\n",
      "Episode 14 Step 2908 Action [[1.07722302]] Reward -6.929554494005886 Loss 0.4507386088371277\n",
      "Episode 14 Step 2909 Action [[1.30124161]] Reward -8.056707213109755 Loss 6.4908766746521\n",
      "Episode 14 Step 2910 Action [[1.32073991]] Reward -9.213275591588847 Loss 0.21347683668136597\n",
      "Episode 14 Step 2911 Action [[1.6782239]] Reward -10.365330035963146 Loss 0.21070300042629242\n",
      "Episode 14 Step 2912 Action [[1.42108294]] Reward -11.077509870708345 Loss 0.2714371085166931\n",
      "Episode 14 Step 2913 Action [[1.27343301]] Reward -9.860953337200927 Loss 0.17195339500904083\n",
      "Episode 14 Step 2914 Action [[0.95760491]] Reward -8.741271974604105 Loss 6.296436786651611\n",
      "Episode 14 Step 2915 Action [[0.10334071]] Reward -7.764134463104394 Loss 0.2958003878593445\n",
      "Episode 14 Step 2916 Action [[-0.35620306]] Reward -6.967408504201302 Loss 2.042804718017578\n",
      "Episode 14 Step 2917 Action [[-0.97408656]] Reward -6.314934375696533 Loss 0.5422840118408203\n",
      "Episode 14 Step 2918 Action [[-0.62635603]] Reward -5.799882406094472 Loss 0.2506292462348938\n",
      "Episode 14 Step 2919 Action [[-0.64920714]] Reward -5.397988676643647 Loss 0.1253652274608612\n",
      "Episode 14 Step 2920 Action [[-1.0755612]] Reward -5.1428939955221535 Loss 0.13463124632835388\n",
      "Episode 14 Step 2921 Action [[-1.11381369]] Reward -5.029398183502929 Loss 4.1826276779174805\n",
      "Episode 14 Step 2922 Action [[-0.88757538]] Reward -5.045197540586366 Loss 0.3456583023071289\n",
      "Episode 14 Step 2923 Action [[-0.88395076]] Reward -5.202759707887131 Loss 0.4858526885509491\n",
      "Episode 14 Step 2924 Action [[-0.79235476]] Reward -5.501771266219691 Loss 2.8153297901153564\n",
      "Episode 14 Step 2925 Action [[-0.49316517]] Reward -5.9431211797139625 Loss 0.24434171617031097\n",
      "Episode 14 Step 2926 Action [[0.54551773]] Reward -6.54218887035661 Loss 0.109157994389534\n",
      "Episode 14 Step 2927 Action [[0.90943198]] Reward -7.384599053572935 Loss 0.1297575831413269\n",
      "Episode 14 Step 2928 Action [[1.26371738]] Reward -8.443945844492706 Loss 0.27592161297798157\n",
      "Episode 14 Step 2929 Action [[1.15922225]] Reward -9.705351598134211 Loss 0.12798503041267395\n",
      "Episode 14 Step 2930 Action [[1.37950045]] Reward -11.069783095666702 Loss 0.200946643948555\n",
      "Episode 14 Step 2931 Action [[1.29020678]] Reward -10.41491058495532 Loss 1.8941998481750488\n",
      "Episode 14 Step 2932 Action [[1.11210372]] Reward -9.312183314454062 Loss 0.1273796111345291\n",
      "Episode 14 Step 2933 Action [[1.02546905]] Reward -8.180346615341637 Loss 0.0791630670428276\n",
      "Episode 14 Step 2934 Action [[1.09708644]] Reward -7.078974218705619 Loss 0.059357933700084686\n",
      "Episode 14 Step 2935 Action [[0.79616217]] Reward -6.066987403025846 Loss 0.0635417252779007\n",
      "Episode 14 Step 2936 Action [[0.85948725]] Reward -5.158051440799135 Loss 0.06940370053052902\n",
      "Episode 14 Step 2937 Action [[0.49438941]] Reward -4.413574057966593 Loss 0.1022375151515007\n",
      "Episode 14 Step 2938 Action [[0.25177161]] Reward -3.834088506265568 Loss 3.5721330642700195\n",
      "Episode 14 Step 2939 Action [[-0.03433879]] Reward -3.455796613531839 Loss 0.12098338454961777\n",
      "Episode 14 Step 2940 Action [[0.26833776]] Reward -3.3020438480673717 Loss 0.14289399981498718\n",
      "Episode 14 Step 2941 Action [[0.72710574]] Reward -3.377229534917982 Loss 0.15040862560272217\n",
      "Episode 14 Step 2942 Action [[0.65163015]] Reward -3.647100279402316 Loss 0.08507151901721954\n",
      "Episode 14 Step 2943 Action [[0.8453014]] Reward -4.118268919674737 Loss 0.13177275657653809\n",
      "Episode 14 Step 2944 Action [[0.6209869]] Reward -4.772194102854799 Loss 0.11410420387983322\n",
      "Episode 14 Step 2945 Action [[0.97190648]] Reward -5.637770816042147 Loss 0.07974854111671448\n",
      "Episode 14 Step 2946 Action [[1.12794917]] Reward -6.641612760880734 Loss 0.05850846320390701\n",
      "Episode 14 Step 2947 Action [[0.99011151]] Reward -7.75363723883121 Loss 0.08530322462320328\n",
      "Episode 14 Step 2948 Action [[0.73299754]] Reward -8.963921284459996 Loss 0.09349431097507477\n",
      "Episode 14 Step 2949 Action [[0.5636585]] Reward -10.245591199123657 Loss 1.8692306280136108\n",
      "Episode 14 Step 2950 Action [[0.51007127]] Reward -11.521048964347116 Loss 0.12423938512802124\n",
      "Episode 14 Step 2951 Action [[0.76983553]] Reward -10.243807170828648 Loss 0.10903021693229675\n",
      "Episode 14 Step 2952 Action [[-0.31773481]] Reward -8.976016472905625 Loss 0.09381271153688431\n",
      "Episode 14 Step 2953 Action [[-1.37523138]] Reward -7.875380099926278 Loss 0.02930082008242607\n",
      "Episode 14 Step 2954 Action [[-1.28333207]] Reward -6.9192752773972375 Loss 0.056910641491413116\n",
      "Episode 14 Step 2955 Action [[-1.08353103]] Reward -6.025333780139289 Loss 0.05470381677150726\n",
      "Episode 14 Step 2956 Action [[-0.87771511]] Reward -5.226205502339072 Loss 0.09620226919651031\n",
      "Episode 14 Step 2957 Action [[-0.77682543]] Reward -4.5590134397396325 Loss 5.342992305755615\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 14 Step 2958 Action [[-0.9289855]] Reward -4.055525551159754 Loss 0.19627532362937927\n",
      "Episode 14 Step 2959 Action [[-0.61731014]] Reward -3.7297001148535087 Loss 0.27857670187950134\n",
      "Episode 14 Step 2960 Action [[-0.78725522]] Reward -3.58186762903097 Loss 0.25227752327919006\n",
      "Episode 14 Step 2961 Action [[-0.9166556]] Reward -3.6214198901928323 Loss 2.7312583923339844\n",
      "Episode 14 Step 2962 Action [[-1.37607519]] Reward -3.839285505794314 Loss 0.047824759036302567\n",
      "Episode 14 Step 2963 Action [[-1.70835033]] Reward -4.2027330237653215 Loss 0.37457311153411865\n",
      "Episode 14 Step 2964 Action [[-1.67753243]] Reward -4.685929710179027 Loss 6.98115873336792\n",
      "Episode 14 Step 2965 Action [[-1.39677255]] Reward -5.295552775421704 Loss 0.2209758758544922\n",
      "Episode 14 Step 2966 Action [[-1.53665426]] Reward -6.05202941457491 Loss 0.6625912189483643\n",
      "Episode 14 Step 2967 Action [[-0.30693649]] Reward -6.899895921376577 Loss 0.555781900882721\n",
      "Episode 14 Step 2968 Action [[0.62936213]] Reward -7.98918468966855 Loss 2.7383759021759033\n",
      "Episode 14 Step 2969 Action [[0.90389087]] Reward -9.325483207561714 Loss 5.959333419799805\n",
      "Episode 14 Step 2970 Action [[0.69507051]] Reward -10.814101910882847 Loss 0.24207666516304016\n",
      "Episode 14 Step 2971 Action [[1.05231712]] Reward -11.121024615769368 Loss 0.0945238471031189\n",
      "Episode 14 Step 2972 Action [[0.9662804]] Reward -9.910448816146177 Loss 0.2056897133588791\n",
      "Episode 14 Step 2973 Action [[1.01915428]] Reward -8.668429336978893 Loss 0.2067873477935791\n",
      "Episode 14 Step 2974 Action [[0.91387667]] Reward -7.4592505282229835 Loss 0.05979101359844208\n",
      "Episode 14 Step 2975 Action [[0.72452093]] Reward -6.318827796439449 Loss 0.2670772075653076\n",
      "Episode 14 Step 2976 Action [[0.79764028]] Reward -5.293425910149739 Loss 2.823914051055908\n",
      "Episode 14 Step 2977 Action [[0.99356318]] Reward -4.441604997447923 Loss 0.16221484541893005\n",
      "Episode 14 Step 2978 Action [[0.51123935]] Reward -3.7779585057552696 Loss 0.12114157527685165\n",
      "Episode 14 Step 2979 Action [[0.79374102]] Reward -3.28241369254807 Loss 0.11441574990749359\n",
      "Episode 14 Step 2980 Action [[0.93967401]] Reward -2.9917434957739784 Loss 0.18954963982105255\n",
      "Episode 14 Step 2981 Action [[1.0499819]] Reward -2.883239691239971 Loss 5.077589988708496\n",
      "Episode 14 Step 2982 Action [[0.82593532]] Reward -2.945438539944026 Loss 0.04752451926469803\n",
      "Episode 14 Step 2983 Action [[0.90894688]] Reward -3.191544183921344 Loss 0.08846111595630646\n",
      "Episode 14 Step 2984 Action [[1.0526596]] Reward -3.622566273063259 Loss 0.13484126329421997\n",
      "Episode 14 Step 2985 Action [[0.92279888]] Reward -4.231468551494471 Loss 0.04995332658290863\n",
      "Episode 14 Step 2986 Action [[1.32633081]] Reward -5.0405636021247515 Loss 0.16571500897407532\n",
      "Episode 14 Step 2987 Action [[1.32941995]] Reward -5.982962563707494 Loss 0.1467747986316681\n",
      "Episode 14 Step 2988 Action [[1.51977276]] Reward -7.064073029910084 Loss 2.2662971019744873\n",
      "Episode 14 Step 2989 Action [[1.40706947]] Reward -8.215957148942142 Loss 0.11381050944328308\n",
      "Episode 14 Step 2990 Action [[1.47243089]] Reward -9.425547611688987 Loss 0.16345350444316864\n",
      "Episode 14 Step 2991 Action [[1.53559578]] Reward -10.611451144256717 Loss 0.11908678710460663\n",
      "Episode 14 Step 2992 Action [[1.24368575]] Reward -11.045033573693193 Loss 0.08245814591646194\n",
      "Episode 14 Step 2993 Action [[1.39296689]] Reward -9.779815153711553 Loss 0.14266033470630646\n",
      "Episode 14 Step 2994 Action [[0.67829245]] Reward -8.58906111050893 Loss 0.053052179515361786\n",
      "Episode 14 Step 2995 Action [[-0.33253779]] Reward -7.578976770838781 Loss 0.059313442558050156\n",
      "Episode 14 Step 2996 Action [[-0.60660639]] Reward -6.75292388737393 Loss 1.0521893501281738\n",
      "Episode 14 Step 2997 Action [[-1.11342707]] Reward -6.054581104096912 Loss 0.22194892168045044\n",
      "Episode 14 Step 2998 Action [[-0.82627173]] Reward -5.4931995445454636 Loss 4.7783379554748535\n",
      "Episode 14 Step 2999 Action [[-0.76082352]] Reward -5.047885010839367 Loss 0.109019935131073\n",
      "TOTAL REWARD @ 14-th Episode  : Reward -1269.4644143728535\n",
      "Total Step: 3000\n",
      "\n",
      "Episode : 15 Replay Buffer 3000\n",
      "[ 0.79020707  0.61283994 -0.87596174]\n",
      "Episode 15 Step 3000 Action [[-1.03917225]] Reward -0.5129482617499065 Loss 0.045667607337236404\n",
      "Episode 15 Step 3001 Action [[-1.05361056]] Reward -0.432062642924604 Loss 0.10643233358860016\n",
      "Episode 15 Step 3002 Action [[-0.99433789]] Reward -0.38952790801422893 Loss 0.09803272783756256\n",
      "Episode 15 Step 3003 Action [[-0.97984215]] Reward -0.381023455984172 Loss 0.07926343381404877\n",
      "Episode 15 Step 3004 Action [[-0.99555879]] Reward -0.40676607094343487 Loss 0.08150738477706909\n",
      "Episode 15 Step 3005 Action [[-1.30912902]] Reward -0.469859224147036 Loss 4.554239749908447\n",
      "Episode 15 Step 3006 Action [[-1.56223531]] Reward -0.5649794164017218 Loss 0.4841764569282532\n",
      "Episode 15 Step 3007 Action [[-0.96042787]] Reward -0.6909888772733682 Loss 0.39779722690582275\n",
      "Episode 15 Step 3008 Action [[-0.99929608]] Reward -0.9011416359960606 Loss 0.041077904403209686\n",
      "Episode 15 Step 3009 Action [[-1.04900542]] Reward -1.1982010929277522 Loss 0.294300377368927\n",
      "Episode 15 Step 3010 Action [[-1.09055575]] Reward -1.6072862569402635 Loss 0.3286266326904297\n",
      "Episode 15 Step 3011 Action [[-1.29113496]] Reward -2.1613350106326408 Loss 0.13361752033233643\n",
      "Episode 15 Step 3012 Action [[-1.52916354]] Reward -2.8779812645331146 Loss 0.13887646794319153\n",
      "Episode 15 Step 3013 Action [[-1.3195833]] Reward -3.77460258247771 Loss 0.2798692584037781\n",
      "Episode 15 Step 3014 Action [[-0.88433214]] Reward -4.931821819183281 Loss 0.23646381497383118\n",
      "Episode 15 Step 3015 Action [[-0.83061586]] Reward -6.4180138622173635 Loss 0.08938062191009521\n",
      "Episode 15 Step 3016 Action [[-0.8745913]] Reward -8.174653963055917 Loss 0.08992158621549606\n",
      "Episode 15 Step 3017 Action [[0.27461746]] Reward -10.11853592465735 Loss 0.1595180481672287\n",
      "Episode 15 Step 3018 Action [[0.37892758]] Reward -12.416055281247111 Loss 0.08563214540481567\n",
      "Episode 15 Step 3019 Action [[0.61208409]] Reward -13.806850983307795 Loss 0.048918869346380234\n",
      "Episode 15 Step 3020 Action [[0.44469911]] Reward -11.918047554271105 Loss 2.006641387939453\n",
      "Episode 15 Step 3021 Action [[0.56111585]] Reward -9.957277806894698 Loss 2.019167900085449\n",
      "Episode 15 Step 3022 Action [[0.2097517]] Reward -8.075675976297545 Loss 1.6692973375320435\n",
      "Episode 15 Step 3023 Action [[-0.48809701]] Reward -6.306921172761112 Loss 0.12720781564712524\n",
      "Episode 15 Step 3024 Action [[-1.10682066]] Reward -4.713855031211436 Loss 0.28093352913856506\n",
      "Episode 15 Step 3025 Action [[-1.18055654]] Reward -3.3921558111604653 Loss 3.250377655029297\n",
      "Episode 15 Step 3026 Action [[-0.99481864]] Reward -2.4199430358363494 Loss 0.09877003729343414\n",
      "Episode 15 Step 3027 Action [[-0.63805788]] Reward -1.775310334835795 Loss 0.11647585034370422\n",
      "Episode 15 Step 3028 Action [[0.06012553]] Reward -1.4045737880133677 Loss 0.15652340650558472\n",
      "Episode 15 Step 3029 Action [[0.55558749]] Reward -1.2486265944580062 Loss 0.20600025355815887\n",
      "Episode 15 Step 3030 Action [[0.96636464]] Reward -1.245471372114078 Loss 0.17941823601722717\n",
      "Episode 15 Step 3031 Action [[1.09043603]] Reward -1.362867903164281 Loss 0.06410495936870575\n",
      "Episode 15 Step 3032 Action [[0.90189761]] Reward -1.5979018001880494 Loss 0.10965228825807571\n",
      "Episode 15 Step 3033 Action [[0.75091761]] Reward -1.9822809610253105 Loss 0.10281489789485931\n",
      "Episode 15 Step 3034 Action [[0.63978276]] Reward -2.5513401070976007 Loss 0.07541050016880035\n",
      "Episode 15 Step 3035 Action [[0.84969294]] Reward -3.343838013628617 Loss 0.046344857662916183\n",
      "Episode 15 Step 3036 Action [[0.90043313]] Reward -4.355798186063089 Loss 0.09683427214622498\n",
      "Episode 15 Step 3037 Action [[1.0342118]] Reward -5.612353458957234 Loss 1.897813081741333\n",
      "Episode 15 Step 3038 Action [[1.25960962]] Reward -7.088155210254246 Loss 0.09904449433088303\n",
      "Episode 15 Step 3039 Action [[1.33097053]] Reward -8.712430030047281 Loss 0.12692409753799438\n",
      "Episode 15 Step 3040 Action [[1.57714806]] Reward -10.423714828619016 Loss 0.08319048583507538\n",
      "Episode 15 Step 3041 Action [[1.61181506]] Reward -12.081066564871719 Loss 13.15792465209961\n",
      "Episode 15 Step 3042 Action [[1.70510434]] Reward -11.974930834247619 Loss 0.6197406053543091\n",
      "Episode 15 Step 3043 Action [[1.53092031]] Reward -10.15146404331756 Loss 1.2599430084228516\n",
      "Episode 15 Step 3044 Action [[0.47834595]] Reward -8.463045017170904 Loss 1.715121865272522\n",
      "Episode 15 Step 3045 Action [[0.07752599]] Reward -7.054421477737491 Loss 0.2130555808544159\n",
      "Episode 15 Step 3046 Action [[-0.1278743]] Reward -5.858478181152823 Loss 0.2554643154144287\n",
      "Episode 15 Step 3047 Action [[-0.68657989]] Reward -4.876939487005586 Loss 0.5758140087127686\n",
      "Episode 15 Step 3048 Action [[-0.79374792]] Reward -4.131982920518181 Loss 0.2812242805957794\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 15 Step 3049 Action [[-0.93010567]] Reward -3.580207649974484 Loss 0.13617421686649323\n",
      "Episode 15 Step 3050 Action [[-1.17438197]] Reward -3.2161795773355895 Loss 0.3955117464065552\n",
      "Episode 15 Step 3051 Action [[-1.41934628]] Reward -3.0269621135401215 Loss 0.2555351257324219\n",
      "Episode 15 Step 3052 Action [[-0.81463714]] Reward -2.990755616950245 Loss 0.1892678439617157\n",
      "Episode 15 Step 3053 Action [[-0.91212865]] Reward -3.1295027276621785 Loss 0.1290433406829834\n",
      "Episode 15 Step 3054 Action [[-1.13672624]] Reward -3.450553048558726 Loss 0.18945622444152832\n",
      "Episode 15 Step 3055 Action [[-1.16107073]] Reward -3.9402587237528532 Loss 0.21985119581222534\n",
      "Episode 15 Step 3056 Action [[-1.26648088]] Reward -4.604290895690904 Loss 1.2342429161071777\n",
      "Episode 15 Step 3057 Action [[-1.14956166]] Reward -5.428026693537772 Loss 0.06670596450567245\n",
      "Episode 15 Step 3058 Action [[-0.32563353]] Reward -6.418644065676731 Loss 0.3000648021697998\n",
      "Episode 15 Step 3059 Action [[0.42723929]] Reward -7.664110970140156 Loss 0.2053285390138626\n",
      "Episode 15 Step 3060 Action [[0.62852528]] Reward -9.174440227046137 Loss 0.11546622216701508\n",
      "Episode 15 Step 3061 Action [[0.57426695]] Reward -10.844567943685124 Loss 0.0868498682975769\n",
      "Episode 15 Step 3062 Action [[0.33953921]] Reward -11.919789835292818 Loss 8.849052429199219\n",
      "Episode 15 Step 3063 Action [[0.28227018]] Reward -10.484475008402542 Loss 0.10139564424753189\n",
      "Episode 15 Step 3064 Action [[0.3676645]] Reward -9.03331092566766 Loss 6.7125630378723145\n",
      "Episode 15 Step 3065 Action [[0.34378351]] Reward -7.6438378350578615 Loss 1.1753352880477905\n",
      "Episode 15 Step 3066 Action [[0.41011467]] Reward -6.363779682555107 Loss 0.9536215662956238\n",
      "Episode 15 Step 3067 Action [[0.43357173]] Reward -5.25296860998085 Loss 0.18243306875228882\n",
      "Episode 15 Step 3068 Action [[0.47535567]] Reward -4.33817369741161 Loss 0.47065603733062744\n",
      "Episode 15 Step 3069 Action [[0.67658482]] Reward -3.634829926496486 Loss 4.253007411956787\n",
      "Episode 15 Step 3070 Action [[0.72016752]] Reward -3.1456897404361794 Loss 1.9096978902816772\n",
      "Episode 15 Step 3071 Action [[0.79120425]] Reward -2.8517219570953944 Loss 0.29804325103759766\n",
      "Episode 15 Step 3072 Action [[1.08430055]] Reward -2.745711093019215 Loss 0.7506616115570068\n",
      "Episode 15 Step 3073 Action [[1.2696043]] Reward -2.812402450747035 Loss 1.156510353088379\n",
      "Episode 15 Step 3074 Action [[1.20593357]] Reward -3.036839970675352 Loss 0.7427493333816528\n",
      "Episode 15 Step 3075 Action [[1.5022486]] Reward -3.42941963620677 Loss 0.5884228944778442\n",
      "Episode 15 Step 3076 Action [[1.36832938]] Reward -3.9678038343522446 Loss 0.7646018266677856\n",
      "Episode 15 Step 3077 Action [[1.67074988]] Reward -4.679928655709656 Loss 0.4235376715660095\n",
      "Episode 15 Step 3078 Action [[1.28914373]] Reward -5.516668674433907 Loss 0.12398474663496017\n",
      "Episode 15 Step 3079 Action [[1.19401706]] Reward -6.5349138593127085 Loss 0.40820667147636414\n",
      "Episode 15 Step 3080 Action [[0.94677012]] Reward -7.689661143879372 Loss 0.4560000002384186\n",
      "Episode 15 Step 3081 Action [[1.05640475]] Reward -8.967357076792752 Loss 0.22977247834205627\n",
      "Episode 15 Step 3082 Action [[0.93406705]] Reward -10.263845838339526 Loss 3.6437501907348633\n",
      "Episode 15 Step 3083 Action [[0.80737936]] Reward -11.549243188432321 Loss 0.4024905562400818\n",
      "Episode 15 Step 3084 Action [[0.68184595]] Reward -10.356079566093207 Loss 0.10046614706516266\n",
      "Episode 15 Step 3085 Action [[0.67435255]] Reward -9.070663001851738 Loss 0.21924175322055817\n",
      "Episode 15 Step 3086 Action [[0.38606258]] Reward -7.872872467155168 Loss 0.350594699382782\n",
      "Episode 15 Step 3087 Action [[-0.95125449]] Reward -6.831131709685015 Loss 0.16562587022781372\n",
      "Episode 15 Step 3088 Action [[-0.94616517]] Reward -6.010347298488565 Loss 0.31448304653167725\n",
      "Episode 15 Step 3089 Action [[-0.92149904]] Reward -5.306162675323877 Loss 7.285391330718994\n",
      "Episode 15 Step 3090 Action [[-0.748242]] Reward -4.739699241839383 Loss 0.0776652842760086\n",
      "Episode 15 Step 3091 Action [[-0.71989443]] Reward -4.3260927387890815 Loss 0.5597719550132751\n",
      "Episode 15 Step 3092 Action [[-0.8201679]] Reward -4.085255626115264 Loss 1.0927557945251465\n",
      "Episode 15 Step 3093 Action [[-0.75537761]] Reward -4.019785642418459 Loss 0.7947999835014343\n",
      "Episode 15 Step 3094 Action [[-0.97843304]] Reward -4.131877077242613 Loss 0.34482020139694214\n",
      "Episode 15 Step 3095 Action [[-1.01740759]] Reward -4.4094817554318 Loss 0.5847082138061523\n",
      "Episode 15 Step 3096 Action [[-1.31957318]] Reward -4.848773323878341 Loss 1.4983863830566406\n",
      "Episode 15 Step 3097 Action [[-1.25070563]] Reward -5.415996238568723 Loss 2.266105890274048\n",
      "Episode 15 Step 3098 Action [[-0.94557929]] Reward -6.116629111899151 Loss 0.6845626831054688\n",
      "Episode 15 Step 3099 Action [[-0.8220532]] Reward -6.965268352186033 Loss 1.208416223526001\n",
      "Episode 15 Step 3100 Action [[0.21802968]] Reward -7.93052671697992 Loss 1.1582368612289429\n",
      "Episode 15 Step 3101 Action [[0.87136162]] Reward -9.114176712488787 Loss 0.160223126411438\n",
      "Episode 15 Step 3102 Action [[0.68940995]] Reward -10.480923747044347 Loss 0.4232702851295471\n",
      "Episode 15 Step 3103 Action [[0.64782674]] Reward -11.047821770999962 Loss 1.4069381952285767\n",
      "Episode 15 Step 3104 Action [[0.4537496]] Reward -9.886192124010472 Loss 0.1593369096517563\n",
      "Episode 15 Step 3105 Action [[0.23044186]] Reward -8.699162447395969 Loss 0.23713034391403198\n",
      "Episode 15 Step 3106 Action [[0.4947247]] Reward -7.541471457313396 Loss 0.9154739379882812\n",
      "Episode 15 Step 3107 Action [[0.5966447]] Reward -6.504933388935626 Loss 0.45063844323158264\n",
      "Episode 15 Step 3108 Action [[0.51441146]] Reward -5.604651571903505 Loss 0.20135420560836792\n",
      "Episode 15 Step 3109 Action [[0.55927769]] Reward -4.85733818139217 Loss 1.656305193901062\n",
      "Episode 15 Step 3110 Action [[0.6644058]] Reward -4.2913648937085425 Loss 0.3530624508857727\n",
      "Episode 15 Step 3111 Action [[0.49956103]] Reward -3.913390714375095 Loss 0.2096734344959259\n",
      "Episode 15 Step 3112 Action [[0.31559532]] Reward -3.723855523790414 Loss 0.19867238402366638\n",
      "Episode 15 Step 3113 Action [[0.37320944]] Reward -3.7400532267462676 Loss 0.28689974546432495\n",
      "Episode 15 Step 3114 Action [[0.73164982]] Reward -3.967482553965556 Loss 0.2522570490837097\n",
      "Episode 15 Step 3115 Action [[0.83294515]] Reward -4.380318413964369 Loss 0.20156055688858032\n",
      "Episode 15 Step 3116 Action [[1.02050541]] Reward -4.971171365556219 Loss 0.19072872400283813\n",
      "Episode 15 Step 3117 Action [[1.07662313]] Reward -5.715410341249836 Loss 0.23466703295707703\n",
      "Episode 15 Step 3118 Action [[1.0673582]] Reward -6.598414708773709 Loss 0.2342558354139328\n",
      "Episode 15 Step 3119 Action [[1.40814195]] Reward -7.598368771034526 Loss 1.8638997077941895\n",
      "Episode 15 Step 3120 Action [[1.55468456]] Reward -8.625392492487727 Loss 0.15665991604328156\n",
      "Episode 15 Step 3121 Action [[1.62215108]] Reward -9.641774469573354 Loss 4.539381504058838\n",
      "Episode 15 Step 3122 Action [[1.4930171]] Reward -10.605160559931495 Loss 0.6908707618713379\n",
      "Episode 15 Step 3123 Action [[1.49175205]] Reward -10.313792745641987 Loss 0.5534511804580688\n",
      "Episode 15 Step 3124 Action [[1.80023479]] Reward -9.269201399150402 Loss 0.33541160821914673\n",
      "Episode 15 Step 3125 Action [[1.12762248]] Reward -8.327304900218156 Loss 0.25686272978782654\n",
      "Episode 15 Step 3126 Action [[0.7865047]] Reward -7.578514072951192 Loss 0.19243581593036652\n",
      "Episode 15 Step 3127 Action [[-0.23816096]] Reward -6.999896011433126 Loss 0.16083639860153198\n",
      "Episode 15 Step 3128 Action [[-0.43828025]] Reward -6.588375035803344 Loss 0.25546395778656006\n",
      "Episode 15 Step 3129 Action [[-0.74253925]] Reward -6.297360170662512 Loss 0.23948383331298828\n",
      "Episode 15 Step 3130 Action [[-0.72067718]] Reward -6.121416950091177 Loss 0.22103044390678406\n",
      "Episode 15 Step 3131 Action [[-0.96085211]] Reward -6.057957934622256 Loss 0.20032784342765808\n",
      "Episode 15 Step 3132 Action [[-0.94045341]] Reward -6.099594019812155 Loss 0.0848895013332367\n",
      "Episode 15 Step 3133 Action [[-0.42515447]] Reward -6.24312600972803 Loss 0.06034957617521286\n",
      "Episode 15 Step 3134 Action [[0.39392986]] Reward -6.518821459655977 Loss 0.07104790210723877\n",
      "Episode 15 Step 3135 Action [[0.86429888]] Reward -6.980410941896217 Loss 0.15808334946632385\n",
      "Episode 15 Step 3136 Action [[0.78358546]] Reward -7.645533770381692 Loss 0.060191281139850616\n",
      "Episode 15 Step 3137 Action [[0.92622166]] Reward -8.469005256349494 Loss 5.077796936035156\n",
      "Episode 15 Step 3138 Action [[1.11560698]] Reward -9.436971333631515 Loss 2.942671537399292\n",
      "Episode 15 Step 3139 Action [[0.64900936]] Reward -10.52235021267126 Loss 0.5154312252998352\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 15 Step 3140 Action [[0.73215204]] Reward -10.134299509392227 Loss 0.4349398910999298\n",
      "Episode 15 Step 3141 Action [[0.91424211]] Reward -9.218893787300404 Loss 4.781894683837891\n",
      "Episode 15 Step 3142 Action [[0.76431086]] Reward -8.315141980021009 Loss 0.2232474982738495\n",
      "Episode 15 Step 3143 Action [[1.44552692]] Reward -7.434675602929848 Loss 7.04603385925293\n",
      "Episode 15 Step 3144 Action [[1.60400868]] Reward -6.652807174605574 Loss 0.11349351704120636\n",
      "Episode 15 Step 3145 Action [[1.39362481]] Reward -5.942077624449938 Loss 0.17285123467445374\n",
      "Episode 15 Step 3146 Action [[1.5043048]] Reward -5.308323527159541 Loss 0.23193146288394928\n",
      "Episode 15 Step 3147 Action [[1.49589392]] Reward -4.788427893376684 Loss 0.30144795775413513\n",
      "Episode 15 Step 3148 Action [[1.64553248]] Reward -4.387673758069542 Loss 0.18474844098091125\n",
      "Episode 15 Step 3149 Action [[1.38352437]] Reward -4.1120627790969255 Loss 0.15878328680992126\n",
      "Episode 15 Step 3150 Action [[1.1688355]] Reward -3.963956912371052 Loss 2.2389578819274902\n",
      "Episode 15 Step 3151 Action [[1.18996319]] Reward -3.9634847400533157 Loss 0.0621022954583168\n",
      "Episode 15 Step 3152 Action [[1.28793323]] Reward -4.116581894667185 Loss 0.09099562466144562\n",
      "Episode 15 Step 3153 Action [[0.8845467]] Reward -4.415849346652033 Loss 4.581955432891846\n",
      "Episode 15 Step 3154 Action [[1.06719438]] Reward -4.894198192781089 Loss 0.6146347522735596\n",
      "Episode 15 Step 3155 Action [[1.56699423]] Reward -5.521524132751809 Loss 0.5921410322189331\n",
      "Episode 15 Step 3156 Action [[1.56009135]] Reward -6.234378481213174 Loss 0.2899739742279053\n",
      "Episode 15 Step 3157 Action [[1.19845312]] Reward -7.03769348340813 Loss 0.2948524057865143\n",
      "Episode 15 Step 3158 Action [[1.20666603]] Reward -7.950303481972863 Loss 0.2699495553970337\n",
      "Episode 15 Step 3159 Action [[1.34021019]] Reward -8.907018182872433 Loss 0.1319626122713089\n",
      "Episode 15 Step 3160 Action [[1.38716651]] Reward -9.847056084306617 Loss 0.18588238954544067\n",
      "Episode 15 Step 3161 Action [[1.40476195]] Reward -10.7347565459405 Loss 0.0923801064491272\n",
      "Episode 15 Step 3162 Action [[1.36274384]] Reward -9.897892270771784 Loss 0.3197743892669678\n",
      "Episode 15 Step 3163 Action [[1.38852757]] Reward -8.973957229920728 Loss 0.27430009841918945\n",
      "Episode 15 Step 3164 Action [[0.86711745]] Reward -8.16700287334045 Loss 0.15080022811889648\n",
      "Episode 15 Step 3165 Action [[0.26049441]] Reward -7.52348718458714 Loss 0.13727734982967377\n",
      "Episode 15 Step 3166 Action [[-0.40248294]] Reward -7.033222361859785 Loss 2.7000348567962646\n",
      "Episode 15 Step 3167 Action [[-0.15163281]] Reward -6.672125129220155 Loss 2.836731433868408\n",
      "Episode 15 Step 3168 Action [[-0.20837974]] Reward -6.424471949902038 Loss 7.473424434661865\n",
      "Episode 15 Step 3169 Action [[-0.01391765]] Reward -6.309330965792508 Loss 0.947076678276062\n",
      "Episode 15 Step 3170 Action [[-0.28511288]] Reward -6.33854626728239 Loss 2.462099075317383\n",
      "Episode 15 Step 3171 Action [[0.10985195]] Reward -6.50177601247425 Loss 0.7066326141357422\n",
      "Episode 15 Step 3172 Action [[0.27488047]] Reward -6.819251355597071 Loss 0.17543251812458038\n",
      "Episode 15 Step 3173 Action [[0.81870639]] Reward -7.293446916868987 Loss 0.8009740710258484\n",
      "Episode 15 Step 3174 Action [[1.05130049]] Reward -7.956481073218152 Loss 0.9563636183738708\n",
      "Episode 15 Step 3175 Action [[1.26556536]] Reward -8.793047252412027 Loss 0.5055336952209473\n",
      "Episode 15 Step 3176 Action [[1.51438225]] Reward -9.784376889825205 Loss 0.1609591841697693\n",
      "Episode 15 Step 3177 Action [[1.46429061]] Reward -10.675651079854436 Loss 0.4607833921909332\n",
      "Episode 15 Step 3178 Action [[1.39031231]] Reward -9.819560474621602 Loss 2.9565482139587402\n",
      "Episode 15 Step 3179 Action [[1.30704813]] Reward -8.914601733761845 Loss 0.5625060796737671\n",
      "Episode 15 Step 3180 Action [[1.31695503]] Reward -7.993323851707111 Loss 0.3104932904243469\n",
      "Episode 15 Step 3181 Action [[1.26692242]] Reward -7.098234794582936 Loss 0.555893063545227\n",
      "Episode 15 Step 3182 Action [[1.18514591]] Reward -6.260060266044699 Loss 1.527117133140564\n",
      "Episode 15 Step 3183 Action [[1.267926]] Reward -5.511251200652712 Loss 0.2833254933357239\n",
      "Episode 15 Step 3184 Action [[1.07524892]] Reward -4.884640203555812 Loss 0.07897917926311493\n",
      "Episode 15 Step 3185 Action [[0.9937076]] Reward -4.387079138215582 Loss 0.5461654663085938\n",
      "Episode 15 Step 3186 Action [[1.3301227]] Reward -4.043099388723107 Loss 2.4726223945617676\n",
      "Episode 15 Step 3187 Action [[1.27630877]] Reward -3.856395769415694 Loss 0.07358933985233307\n",
      "Episode 15 Step 3188 Action [[1.3078718]] Reward -3.8165561891843787 Loss 0.06740479916334152\n",
      "Episode 15 Step 3189 Action [[1.11322256]] Reward -3.9246494169627386 Loss 0.19199183583259583\n",
      "Episode 15 Step 3190 Action [[1.30074694]] Reward -4.194050177515582 Loss 1.7201967239379883\n",
      "Episode 15 Step 3191 Action [[1.39186436]] Reward -4.609675086366035 Loss 0.06744121760129929\n",
      "Episode 15 Step 3192 Action [[1.34665425]] Reward -5.162260104726949 Loss 0.10390087962150574\n",
      "Episode 15 Step 3193 Action [[1.35529637]] Reward -5.8506107144519435 Loss 0.11089733242988586\n",
      "Episode 15 Step 3194 Action [[1.37278097]] Reward -6.654786654605503 Loss 0.0894593596458435\n",
      "Episode 15 Step 3195 Action [[1.07452988]] Reward -7.546143067201346 Loss 0.10548833757638931\n",
      "Episode 15 Step 3196 Action [[1.20544154]] Reward -8.533172990889668 Loss 10.11958122253418\n",
      "Episode 15 Step 3197 Action [[1.13889644]] Reward -9.529254958529334 Loss 0.2281964123249054\n",
      "Episode 15 Step 3198 Action [[1.52151003]] Reward -10.512398977770767 Loss 0.5287255644798279\n",
      "Episode 15 Step 3199 Action [[1.4771284]] Reward -10.30627738690894 Loss 0.47955453395843506\n",
      "Now we save model\n",
      "TOTAL REWARD @ 15-th Episode  : Reward -1235.514814611385\n",
      "Total Step: 3200\n",
      "\n",
      "Episode : 16 Replay Buffer 3200\n",
      "[-0.22393544 -0.97460398  0.16232754]\n",
      "Episode 16 Step 3200 Action [[1.77535739]] Reward -3.23372709955729 Loss 0.23975466191768646\n",
      "Episode 16 Step 3201 Action [[1.55535544]] Reward -3.294044219490515 Loss 0.29745179414749146\n",
      "Episode 16 Step 3202 Action [[1.7173596]] Reward -3.495061375066176 Loss 0.26823657751083374\n",
      "Episode 16 Step 3203 Action [[1.37492382]] Reward -3.8266216670827036 Loss 0.10519160330295563\n",
      "Episode 16 Step 3204 Action [[1.68997914]] Reward -4.3233406199017015 Loss 0.14230340719223022\n",
      "Episode 16 Step 3205 Action [[1.61477263]] Reward -4.94205162212128 Loss 0.21003565192222595\n",
      "Episode 16 Step 3206 Action [[1.80378921]] Reward -5.6974713642245325 Loss 0.2956011891365051\n",
      "Episode 16 Step 3207 Action [[1.42876157]] Reward -6.542753039938782 Loss 0.13082469999790192\n",
      "Episode 16 Step 3208 Action [[1.37070761]] Reward -7.516270728370887 Loss 0.10261323302984238\n",
      "Episode 16 Step 3209 Action [[1.05687634]] Reward -8.55780450754021 Loss 0.16945284605026245\n",
      "Episode 16 Step 3210 Action [[0.87144282]] Reward -9.664087252683453 Loss 0.12470418214797974\n",
      "Episode 16 Step 3211 Action [[0.82605429]] Reward -10.78236045238781 Loss 0.10046760737895966\n",
      "Episode 16 Step 3212 Action [[1.11513118]] Reward -10.429466139831057 Loss 0.08919710665941238\n",
      "Episode 16 Step 3213 Action [[1.00844216]] Reward -9.286864581579756 Loss 0.10237457603216171\n",
      "Episode 16 Step 3214 Action [[0.64785481]] Reward -8.244704545078312 Loss 0.10438427329063416\n",
      "Episode 16 Step 3215 Action [[-0.67099312]] Reward -7.3514064996115955 Loss 0.07027667760848999\n",
      "Episode 16 Step 3216 Action [[-0.87766601]] Reward -6.645309911837929 Loss 0.1054406613111496\n",
      "Episode 16 Step 3217 Action [[-0.93329958]] Reward -6.044844393255043 Loss 2.6162643432617188\n",
      "Episode 16 Step 3218 Action [[-1.1817828]] Reward -5.556837295818372 Loss 0.08150510489940643\n",
      "Episode 16 Step 3219 Action [[-1.18073269]] Reward -5.192418403875977 Loss 0.07614311575889587\n",
      "Episode 16 Step 3220 Action [[-1.32104948]] Reward -4.948967851994958 Loss 0.0667777881026268\n",
      "Episode 16 Step 3221 Action [[-1.46125368]] Reward -4.8297619130610645 Loss 7.883582592010498\n",
      "Episode 16 Step 3222 Action [[-1.42500394]] Reward -4.8282032337948975 Loss 1.9508945941925049\n",
      "Episode 16 Step 3223 Action [[-1.33557582]] Reward -4.944298009609136 Loss 5.421224594116211\n",
      "Episode 16 Step 3224 Action [[-1.52779484]] Reward -5.1833076531661995 Loss 0.4145103394985199\n",
      "Episode 16 Step 3225 Action [[-1.47930839]] Reward -5.525783897565218 Loss 0.2610760033130646\n",
      "Episode 16 Step 3226 Action [[-1.1155361]] Reward -5.970314138294821 Loss 0.1252899318933487\n",
      "Episode 16 Step 3227 Action [[-0.85893802]] Reward -6.538189893693726 Loss 7.990741729736328\n",
      "Episode 16 Step 3228 Action [[-0.54496679]] Reward -7.224870640565535 Loss 0.22515511512756348\n",
      "Episode 16 Step 3229 Action [[0.61647462]] Reward -8.03017570032792 Loss 0.12013103812932968\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 16 Step 3230 Action [[1.00949655]] Reward -9.05146922611125 Loss 0.33267319202423096\n",
      "Episode 16 Step 3231 Action [[1.18845332]] Reward -10.225188129713 Loss 0.23556438088417053\n",
      "Episode 16 Step 3232 Action [[1.15456281]] Reward -10.71194274037989 Loss 1.8797624111175537\n",
      "Episode 16 Step 3233 Action [[1.31384719]] Reward -9.717336600713695 Loss 0.10348212718963623\n",
      "Episode 16 Step 3234 Action [[1.36242761]] Reward -8.705425818110875 Loss 0.13035184144973755\n",
      "Episode 16 Step 3235 Action [[1.27398521]] Reward -7.698836446883503 Loss 0.1934393048286438\n",
      "Episode 16 Step 3236 Action [[1.24409977]] Reward -6.72605378371272 Loss 0.20154808461666107\n",
      "Episode 16 Step 3237 Action [[1.10554895]] Reward -5.832228427974528 Loss 0.047496624290943146\n",
      "Episode 16 Step 3238 Action [[0.72951488]] Reward -5.045127818284407 Loss 0.11008897423744202\n",
      "Episode 16 Step 3239 Action [[1.2789812]] Reward -4.390945484201401 Loss 0.17852497100830078\n",
      "Episode 16 Step 3240 Action [[1.15108796]] Reward -3.9215786251609344 Loss 0.16989603638648987\n",
      "Episode 16 Step 3241 Action [[1.33368599]] Reward -3.603354564164523 Loss 0.07281526178121567\n",
      "Episode 16 Step 3242 Action [[1.09980299]] Reward -3.441661211213064 Loss 0.08326743543148041\n",
      "Episode 16 Step 3243 Action [[1.30619515]] Reward -3.4382830704240823 Loss 0.07101685553789139\n",
      "Episode 16 Step 3244 Action [[1.43270633]] Reward -3.590742380806385 Loss 0.03318922594189644\n",
      "Episode 16 Step 3245 Action [[1.14507189]] Reward -3.8892729450445085 Loss 0.08318552374839783\n",
      "Episode 16 Step 3246 Action [[1.21536661]] Reward -4.360121672334918 Loss 0.08106689900159836\n",
      "Episode 16 Step 3247 Action [[1.24868021]] Reward -4.989208169601792 Loss 0.059168968349695206\n",
      "Episode 16 Step 3248 Action [[1.24958904]] Reward -5.767622383171462 Loss 0.19241154193878174\n",
      "Episode 16 Step 3249 Action [[0.85632298]] Reward -6.678913842687913 Loss 2.5332934856414795\n",
      "Episode 16 Step 3250 Action [[0.79676795]] Reward -7.749957000879777 Loss 0.056765612214803696\n",
      "Episode 16 Step 3251 Action [[0.8569746]] Reward -8.91640147459286 Loss 2.7321255207061768\n",
      "Episode 16 Step 3252 Action [[0.98442538]] Reward -10.110120517900036 Loss 0.17310163378715515\n",
      "Episode 16 Step 3253 Action [[0.85143284]] Reward -11.259615519955046 Loss 3.0367722511291504\n",
      "Episode 16 Step 3254 Action [[0.97323282]] Reward -10.13645953075465 Loss 0.18268966674804688\n",
      "Episode 16 Step 3255 Action [[0.91160681]] Reward -8.96544978686652 Loss 0.26233237981796265\n",
      "Episode 16 Step 3256 Action [[-0.00436885]] Reward -7.90031390353804 Loss 0.19981947541236877\n",
      "Episode 16 Step 3257 Action [[-0.84667411]] Reward -7.019572245393121 Loss 7.211531639099121\n",
      "Episode 16 Step 3258 Action [[-0.61692335]] Reward -6.292410826624273 Loss 0.22018034756183624\n",
      "Episode 16 Step 3259 Action [[-0.72102774]] Reward -5.667530941969253 Loss 0.4557729959487915\n",
      "Episode 16 Step 3260 Action [[-0.67371836]] Reward -5.184679564579281 Loss 0.43199387192726135\n",
      "Episode 16 Step 3261 Action [[-0.64537383]] Reward -4.8513557858526815 Loss 0.16475042700767517\n",
      "Episode 16 Step 3262 Action [[-0.62285978]] Reward -4.679860119488891 Loss 0.2450806200504303\n",
      "Episode 16 Step 3263 Action [[-0.55829082]] Reward -4.677215894352611 Loss 5.3693156242370605\n",
      "Episode 16 Step 3264 Action [[-0.71941803]] Reward -4.849652962974321 Loss 0.4133916199207306\n",
      "Episode 16 Step 3265 Action [[-0.63057579]] Reward -5.185362413548609 Loss 0.5184111595153809\n",
      "Episode 16 Step 3266 Action [[-0.57632838]] Reward -5.687152119986664 Loss 2.677637815475464\n",
      "Episode 16 Step 3267 Action [[-0.27550057]] Reward -6.347257035500656 Loss 3.997480630874634\n",
      "Episode 16 Step 3268 Action [[0.5249756]] Reward -7.1793142759085296 Loss 0.17028409242630005\n",
      "Episode 16 Step 3269 Action [[1.34427245]] Reward -8.243487235789457 Loss 0.2060890644788742\n",
      "Episode 16 Step 3270 Action [[1.36101661]] Reward -9.56505704849707 Loss 0.17421330511569977\n",
      "Episode 16 Step 3271 Action [[1.24553652]] Reward -11.025956917811927 Loss 0.3014184236526489\n",
      "Episode 16 Step 3272 Action [[1.46465783]] Reward -10.69229189018638 Loss 0.13884441554546356\n",
      "Episode 16 Step 3273 Action [[1.33770825]] Reward -9.554643629606876 Loss 0.15153944492340088\n",
      "Episode 16 Step 3274 Action [[1.33107177]] Reward -8.378235574108267 Loss 0.09714067727327347\n",
      "Episode 16 Step 3275 Action [[1.18191368]] Reward -7.22386272648697 Loss 3.5977983474731445\n",
      "Episode 16 Step 3276 Action [[1.26145532]] Reward -6.129411964272949 Loss 0.10044987499713898\n",
      "Episode 16 Step 3277 Action [[1.0429277]] Reward -5.159102982321431 Loss 1.5932834148406982\n",
      "Episode 16 Step 3278 Action [[1.21515034]] Reward -4.321999482680728 Loss 0.32201892137527466\n",
      "Episode 16 Step 3279 Action [[1.01876549]] Reward -3.662114584654217 Loss 0.3996199369430542\n",
      "Episode 16 Step 3280 Action [[1.34841748]] Reward -3.164585535810297 Loss 0.20192112028598785\n",
      "Episode 16 Step 3281 Action [[1.22506599]] Reward -2.842802137335884 Loss 0.10279344022274017\n",
      "Episode 16 Step 3282 Action [[1.3057318]] Reward -2.6743206462036464 Loss 0.09800265729427338\n",
      "Episode 16 Step 3283 Action [[1.43982037]] Reward -2.6600331609157193 Loss 0.17918571829795837\n",
      "Episode 16 Step 3284 Action [[1.34377856]] Reward -2.7915501961209226 Loss 0.2198181003332138\n",
      "Episode 16 Step 3285 Action [[1.48759766]] Reward -3.0780170159834053 Loss 0.1551089584827423\n",
      "Episode 16 Step 3286 Action [[1.57141993]] Reward -3.514128148327574 Loss 2.6511831283569336\n",
      "Episode 16 Step 3287 Action [[1.518308]] Reward -4.100676388735273 Loss 0.2671235203742981\n",
      "Episode 16 Step 3288 Action [[1.41115741]] Reward -4.84883725076094 Loss 0.2741023302078247\n",
      "Episode 16 Step 3289 Action [[1.66533498]] Reward -5.764759902987229 Loss 0.09896554797887802\n",
      "Episode 16 Step 3290 Action [[1.69461536]] Reward -6.785768100706623 Loss 0.3615637421607971\n",
      "Episode 16 Step 3291 Action [[1.32835338]] Reward -7.89340550007863 Loss 0.25850504636764526\n",
      "Episode 16 Step 3292 Action [[1.32462206]] Reward -9.103839307177498 Loss 0.22228895127773285\n",
      "Episode 16 Step 3293 Action [[1.10722068]] Reward -10.321154941337607 Loss 5.648423671722412\n",
      "Episode 16 Step 3294 Action [[0.91841577]] Reward -11.423593911902408 Loss 6.881931781768799\n",
      "Episode 16 Step 3295 Action [[0.94213077]] Reward -10.138280344078629 Loss 0.9558165669441223\n",
      "Episode 16 Step 3296 Action [[0.33368]] Reward -8.90363890332631 Loss 1.1341075897216797\n",
      "Episode 16 Step 3297 Action [[-1.12156842]] Reward -7.812521839032793 Loss 0.3518277704715729\n",
      "Episode 16 Step 3298 Action [[-1.07847203]] Reward -6.91253769223121 Loss 0.7881380915641785\n",
      "Episode 16 Step 3299 Action [[-1.10932555]] Reward -6.086599594090654 Loss 0.5215896368026733\n",
      "Episode 16 Step 3300 Action [[-1.31261175]] Reward -5.369845362946898 Loss 6.616654872894287\n",
      "Episode 16 Step 3301 Action [[-1.43306586]] Reward -4.786086012992796 Loss 0.3456292748451233\n",
      "Episode 16 Step 3302 Action [[-1.39732099]] Reward -4.334615603034577 Loss 1.178205966949463\n",
      "Episode 16 Step 3303 Action [[-1.35428293]] Reward -4.0158543764936345 Loss 0.7400379180908203\n",
      "Episode 16 Step 3304 Action [[-1.32544555]] Reward -3.836949032099612 Loss 0.8738610148429871\n",
      "Episode 16 Step 3305 Action [[-1.46411595]] Reward -3.803374405886137 Loss 0.3589628338813782\n",
      "Episode 16 Step 3306 Action [[-1.23840354]] Reward -3.9099962131873234 Loss 0.4772372245788574\n",
      "Episode 16 Step 3307 Action [[-1.21858539]] Reward -4.170156420362209 Loss 0.5291211009025574\n",
      "Episode 16 Step 3308 Action [[-1.4724187]] Reward -4.585619384415954 Loss 0.4079468250274658\n",
      "Episode 16 Step 3309 Action [[-1.53547188]] Reward -5.128936024083708 Loss 0.32688266038894653\n",
      "Episode 16 Step 3310 Action [[-1.32891015]] Reward -5.791946320092105 Loss 0.6115415096282959\n",
      "Episode 16 Step 3311 Action [[-0.7595152]] Reward -6.585048662771166 Loss 1.476546049118042\n",
      "Episode 16 Step 3312 Action [[0.28627762]] Reward -7.545139162282282 Loss 0.20423123240470886\n",
      "Episode 16 Step 3313 Action [[0.76370928]] Reward -8.746226651798057 Loss 6.174901962280273\n",
      "Episode 16 Step 3314 Action [[0.73461613]] Reward -10.129578656678376 Loss 0.3245784640312195\n",
      "Episode 16 Step 3315 Action [[0.72185144]] Reward -11.513287568399603 Loss 0.8664976358413696\n",
      "Episode 16 Step 3316 Action [[0.92351169]] Reward -10.327045008913426 Loss 0.6136378645896912\n",
      "Episode 16 Step 3317 Action [[1.15119363]] Reward -9.130359316534905 Loss 0.7485953569412231\n",
      "Episode 16 Step 3318 Action [[1.51228717]] Reward -7.962015096030613 Loss 0.4872743785381317\n",
      "Episode 16 Step 3319 Action [[1.32023243]] Reward -6.8675412627277215 Loss 0.1706797331571579\n",
      "Episode 16 Step 3320 Action [[1.19630575]] Reward -5.833656099470709 Loss 0.27283942699432373\n",
      "Episode 16 Step 3321 Action [[1.39301426]] Reward -4.913544191332305 Loss 0.8749608993530273\n",
      "Episode 16 Step 3322 Action [[1.37992871]] Reward -4.154065869931214 Loss 0.7107644081115723\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 16 Step 3323 Action [[1.28435954]] Reward -3.5464941561170136 Loss 0.13954918086528778\n",
      "Episode 16 Step 3324 Action [[1.08681872]] Reward -3.092664919281918 Loss 0.2724537253379822\n",
      "Episode 16 Step 3325 Action [[1.17983017]] Reward -2.7973781522962433 Loss 0.26016417145729065\n",
      "Episode 16 Step 3326 Action [[1.45936962]] Reward -2.667397231945801 Loss 0.1950104534626007\n",
      "Episode 16 Step 3327 Action [[1.4859266]] Reward -2.68700495631572 Loss 0.3514889180660248\n",
      "Episode 16 Step 3328 Action [[1.40349131]] Reward -2.8491778277276665 Loss 0.32122647762298584\n",
      "Episode 16 Step 3329 Action [[1.34848704]] Reward -3.163806653843465 Loss 0.23291844129562378\n",
      "Episode 16 Step 3330 Action [[1.26996413]] Reward -3.641476248704548 Loss 0.0802944228053093\n",
      "Episode 16 Step 3331 Action [[0.98810387]] Reward -4.294537059481112 Loss 0.1385272890329361\n",
      "Episode 16 Step 3332 Action [[1.02278295]] Reward -5.155518885732672 Loss 0.24241922795772552\n",
      "Episode 16 Step 3333 Action [[0.97460771]] Reward -6.195717428831011 Loss 0.10608232766389847\n",
      "Episode 16 Step 3334 Action [[1.30198419]] Reward -7.401069237653605 Loss 0.12379668653011322\n",
      "Episode 16 Step 3335 Action [[1.28029402]] Reward -8.668445414392725 Loss 0.09321700036525726\n",
      "Episode 16 Step 3336 Action [[1.37002062]] Reward -9.977273565812187 Loss 0.06925281882286072\n",
      "Episode 16 Step 3337 Action [[1.25691945]] Reward -11.243509034337071 Loss 0.07376197725534439\n",
      "Episode 16 Step 3338 Action [[1.55001806]] Reward -10.74262400472477 Loss 0.08882569521665573\n",
      "Episode 16 Step 3339 Action [[1.23725977]] Reward -9.373999578566849 Loss 0.09610729664564133\n",
      "Episode 16 Step 3340 Action [[1.08622621]] Reward -8.149757108167247 Loss 5.728738307952881\n",
      "Episode 16 Step 3341 Action [[-0.27167139]] Reward -7.0927466741152925 Loss 0.2177540510892868\n",
      "Episode 16 Step 3342 Action [[-0.59737796]] Reward -6.281487504316135 Loss 6.77398681640625\n",
      "Episode 16 Step 3343 Action [[-0.58759806]] Reward -5.620553115362802 Loss 0.45292994379997253\n",
      "Episode 16 Step 3344 Action [[-0.75830904]] Reward -5.104854154978231 Loss 0.14483985304832458\n",
      "Episode 16 Step 3345 Action [[-0.47530136]] Reward -4.7491062362362015 Loss 0.1138060986995697\n",
      "Episode 16 Step 3346 Action [[-0.52827094]] Reward -4.5577688703625565 Loss 0.21372731029987335\n",
      "Episode 16 Step 3347 Action [[-0.54645162]] Reward -4.5471709881345985 Loss 0.22599753737449646\n",
      "Episode 16 Step 3348 Action [[-0.60402477]] Reward -4.716998547354774 Loss 0.18335455656051636\n",
      "Episode 16 Step 3349 Action [[-0.71611296]] Reward -5.062370256566023 Loss 0.16670873761177063\n",
      "Episode 16 Step 3350 Action [[-0.95675674]] Reward -5.569407299258167 Loss 0.17511068284511566\n",
      "Episode 16 Step 3351 Action [[-0.96470628]] Reward -6.205305941852629 Loss 0.1446983516216278\n",
      "Episode 16 Step 3352 Action [[-0.65920911]] Reward -6.959479186200259 Loss 7.507486820220947\n",
      "Episode 16 Step 3353 Action [[0.29046953]] Reward -7.843881999077757 Loss 0.2499893456697464\n",
      "Episode 16 Step 3354 Action [[1.06085769]] Reward -8.933040956782033 Loss 6.574184417724609\n",
      "Episode 16 Step 3355 Action [[1.28131736]] Reward -10.224759231644297 Loss 0.224594384431839\n",
      "Episode 16 Step 3356 Action [[1.15160509]] Reward -11.052542050561637 Loss 0.10192494094371796\n",
      "Episode 16 Step 3357 Action [[0.97838664]] Reward -9.970318673934061 Loss 0.10638271272182465\n",
      "Episode 16 Step 3358 Action [[0.91409425]] Reward -8.841169592654218 Loss 0.13875219225883484\n",
      "Episode 16 Step 3359 Action [[0.88625184]] Reward -7.722158465807695 Loss 0.2032167613506317\n",
      "Episode 16 Step 3360 Action [[1.13556843]] Reward -6.665043431116338 Loss 0.06352854520082474\n",
      "Episode 16 Step 3361 Action [[1.16134901]] Reward -5.728728993647419 Loss 0.1079142838716507\n",
      "Episode 16 Step 3362 Action [[1.47644387]] Reward -4.9198735437921775 Loss 0.1353897750377655\n",
      "Episode 16 Step 3363 Action [[1.36495581]] Reward -4.2666639271759985 Loss 0.151175856590271\n",
      "Episode 16 Step 3364 Action [[1.08360801]] Reward -3.749874404418183 Loss 8.219606399536133\n",
      "Episode 16 Step 3365 Action [[1.09160993]] Reward -3.381643248849755 Loss 0.1254352182149887\n",
      "Episode 16 Step 3366 Action [[1.11318888]] Reward -3.180377356449927 Loss 0.21909061074256897\n",
      "Episode 16 Step 3367 Action [[1.25483802]] Reward -3.145031627826808 Loss 2.320188522338867\n",
      "Episode 16 Step 3368 Action [[1.45076624]] Reward -3.2696277287912436 Loss 0.2435767948627472\n",
      "Episode 16 Step 3369 Action [[1.27368671]] Reward -3.5415249529239845 Loss 0.18799400329589844\n",
      "Episode 16 Step 3370 Action [[1.12765367]] Reward -3.9783965695838432 Loss 0.07548470795154572\n",
      "Episode 16 Step 3371 Action [[0.91885827]] Reward -4.592849119140566 Loss 0.04609714820981026\n",
      "Episode 16 Step 3372 Action [[0.82753978]] Reward -5.401097296139202 Loss 0.22149287164211273\n",
      "Episode 16 Step 3373 Action [[0.86001469]] Reward -6.392788649039934 Loss 0.16897989809513092\n",
      "Episode 16 Step 3374 Action [[1.17210308]] Reward -7.531124547175738 Loss 0.08314917981624603\n",
      "Episode 16 Step 3375 Action [[0.89341674]] Reward -8.725094720368137 Loss 0.0812944620847702\n",
      "Episode 16 Step 3376 Action [[0.63852948]] Reward -9.996531911943215 Loss 0.03713618963956833\n",
      "Episode 16 Step 3377 Action [[0.49809104]] Reward -11.297327753914516 Loss 1.8907020092010498\n",
      "Episode 16 Step 3378 Action [[0.41213494]] Reward -10.527011351625083 Loss 6.200595855712891\n",
      "Episode 16 Step 3379 Action [[0.75258303]] Reward -9.264713660187494 Loss 0.19751007854938507\n",
      "Episode 16 Step 3380 Action [[0.94489363]] Reward -8.043743680563825 Loss 0.5927700400352478\n",
      "Episode 16 Step 3381 Action [[-0.33327064]] Reward -6.948652795211786 Loss 0.14678996801376343\n",
      "Episode 16 Step 3382 Action [[-1.19051947]] Reward -6.102557981102454 Loss 0.10530315339565277\n",
      "Episode 16 Step 3383 Action [[-1.1544776]] Reward -5.429052016162997 Loss 0.2365507185459137\n",
      "Episode 16 Step 3384 Action [[-1.21853112]] Reward -4.874852227121789 Loss 0.25343483686447144\n",
      "Episode 16 Step 3385 Action [[-1.26133853]] Reward -4.457234552259951 Loss 0.13034182786941528\n",
      "Episode 16 Step 3386 Action [[-1.24281199]] Reward -4.1802683427910825 Loss 0.10038871318101883\n",
      "Episode 16 Step 3387 Action [[-1.04281779]] Reward -4.046552582839374 Loss 0.24509647488594055\n",
      "Episode 16 Step 3388 Action [[-1.08300813]] Reward -4.067253378638384 Loss 0.2254948616027832\n",
      "Episode 16 Step 3389 Action [[-1.05026102]] Reward -4.246259215851459 Loss 0.11284957826137543\n",
      "Episode 16 Step 3390 Action [[-1.16576556]] Reward -4.585793174408247 Loss 0.1492539644241333\n",
      "Episode 16 Step 3391 Action [[-1.13200658]] Reward -5.072915131089726 Loss 0.12315957248210907\n",
      "Episode 16 Step 3392 Action [[-1.57480871]] Reward -5.707510426132752 Loss 0.09203889220952988\n",
      "Episode 16 Step 3393 Action [[-1.46997283]] Reward -6.422630999228205 Loss 12.157588005065918\n",
      "Episode 16 Step 3394 Action [[-0.72375319]] Reward -7.231066497438821 Loss 0.6182929873466492\n",
      "Episode 16 Step 3395 Action [[0.48075765]] Reward -8.194922862430436 Loss 0.5445559024810791\n",
      "Episode 16 Step 3396 Action [[0.64713576]] Reward -9.391168252452278 Loss 1.3346836566925049\n",
      "Episode 16 Step 3397 Action [[0.62306861]] Reward -10.695927321041486 Loss 0.3330445885658264\n",
      "Episode 16 Step 3398 Action [[0.47025869]] Reward -10.697963527112936 Loss 0.4430834949016571\n",
      "Episode 16 Step 3399 Action [[0.81381516]] Reward -9.562531935864026 Loss 0.6592626571655273\n",
      "TOTAL REWARD @ 16-th Episode  : Reward -1277.5315680716953\n",
      "Total Step: 3400\n",
      "\n",
      "Episode : 17 Replay Buffer 3400\n",
      "[-0.04661982 -0.9989127   0.53683882]\n",
      "Episode 17 Step 3400 Action [[0.27528268]] Reward -2.644985052029114 Loss 0.3764224648475647\n",
      "Episode 17 Step 3401 Action [[0.64488558]] Reward -2.6471713559745873 Loss 0.19875052571296692\n",
      "Episode 17 Step 3402 Action [[0.72904432]] Reward -2.847665070451575 Loss 0.2606315016746521\n",
      "Episode 17 Step 3403 Action [[0.85121048]] Reward -3.2421480267909533 Loss 0.6079388856887817\n",
      "Episode 17 Step 3404 Action [[0.64429442]] Reward -3.8291497556122183 Loss 0.3472413122653961\n",
      "Episode 17 Step 3405 Action [[1.09593496]] Reward -4.643855787397181 Loss 0.08645255118608475\n",
      "Episode 17 Step 3406 Action [[0.73161658]] Reward -5.616795509960188 Loss 0.2475670725107193\n",
      "Episode 17 Step 3407 Action [[0.76257382]] Reward -6.816608342481888 Loss 0.6390773057937622\n",
      "Episode 17 Step 3408 Action [[0.7522407]] Reward -8.173140660948436 Loss 0.37115585803985596\n",
      "Episode 17 Step 3409 Action [[1.0070069]] Reward -9.63463563405167 Loss 0.1963861733675003\n",
      "Episode 17 Step 3410 Action [[0.9059442]] Reward -11.078938593952907 Loss 0.40123555064201355\n",
      "Episode 17 Step 3411 Action [[0.70231736]] Reward -11.489920829500829 Loss 0.2829640209674835\n",
      "Episode 17 Step 3412 Action [[0.41458653]] Reward -10.0335839740967 Loss 5.643767833709717\n",
      "Episode 17 Step 3413 Action [[0.39093932]] Reward -8.655121443499757 Loss 0.200716033577919\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 17 Step 3414 Action [[-0.80998804]] Reward -7.376939616185401 Loss 0.09802992641925812\n",
      "Episode 17 Step 3415 Action [[-1.1261228]] Reward -6.328725962120891 Loss 0.14501048624515533\n",
      "Episode 17 Step 3416 Action [[-0.86088013]] Reward -5.421300436136987 Loss 0.07799286395311356\n",
      "Episode 17 Step 3417 Action [[-0.56800202]] Reward -4.637963391875267 Loss 0.1251562237739563\n",
      "Episode 17 Step 3418 Action [[-0.61666267]] Reward -4.015859931081179 Loss 7.050752639770508\n",
      "Episode 17 Step 3419 Action [[-0.38199721]] Reward -3.589262685291512 Loss 0.33299779891967773\n",
      "Episode 17 Step 3420 Action [[-0.10999893]] Reward -3.359575470022141 Loss 0.4067525863647461\n",
      "Episode 17 Step 3421 Action [[-0.50066361]] Reward -3.349432022852284 Loss 0.2620660364627838\n",
      "Episode 17 Step 3422 Action [[-0.89595242]] Reward -3.551176492088207 Loss 0.10855655372142792\n",
      "Episode 17 Step 3423 Action [[-1.10854622]] Reward -3.933531733229393 Loss 0.24685737490653992\n",
      "Episode 17 Step 3424 Action [[-1.22164413]] Reward -4.482039277474781 Loss 0.36333030462265015\n",
      "Episode 17 Step 3425 Action [[-1.4016054]] Reward -5.188387985416868 Loss 0.19430291652679443\n",
      "Episode 17 Step 3426 Action [[-1.4729226]] Reward -6.0253831063251235 Loss 0.04452061653137207\n",
      "Episode 17 Step 3427 Action [[-0.68565247]] Reward -6.97120403678829 Loss 14.70167064666748\n",
      "Episode 17 Step 3428 Action [[0.64592366]] Reward -8.114006187478676 Loss 1.6653105020523071\n",
      "Episode 17 Step 3429 Action [[0.8203776]] Reward -9.551473180801361 Loss 1.3110294342041016\n",
      "Episode 17 Step 3430 Action [[0.7214898]] Reward -11.12384854288189 Loss 0.6800600290298462\n",
      "Episode 17 Step 3431 Action [[0.80739174]] Reward -11.160429171199024 Loss 0.20374563336372375\n",
      "Episode 17 Step 3432 Action [[0.68506353]] Reward -9.862197260413051 Loss 1.102220892906189\n",
      "Episode 17 Step 3433 Action [[0.81067455]] Reward -8.54279790967924 Loss 1.3875411748886108\n",
      "Episode 17 Step 3434 Action [[0.90275994]] Reward -7.284509159866398 Loss 0.3606926202774048\n",
      "Episode 17 Step 3435 Action [[0.91280599]] Reward -6.131226976387292 Loss 0.28421998023986816\n",
      "Episode 17 Step 3436 Action [[0.86838926]] Reward -5.115700700565856 Loss 1.1308212280273438\n",
      "Episode 17 Step 3437 Action [[0.90030501]] Reward -4.264085286363047 Loss 0.4632088541984558\n",
      "Episode 17 Step 3438 Action [[0.71421822]] Reward -3.596522268643577 Loss 0.1813608855009079\n",
      "Episode 17 Step 3439 Action [[0.48122243]] Reward -3.1096810427667183 Loss 0.6429937481880188\n",
      "Episode 17 Step 3440 Action [[0.69139692]] Reward -2.8160349704613354 Loss 0.6747192144393921\n",
      "Episode 17 Step 3441 Action [[1.22692254]] Reward -2.7227747338475847 Loss 0.23328585922718048\n",
      "Episode 17 Step 3442 Action [[0.96657769]] Reward -2.79734687486263 Loss 0.047703005373477936\n",
      "Episode 17 Step 3443 Action [[1.04704625]] Reward -3.0466196591407093 Loss 0.12267928570508957\n",
      "Episode 17 Step 3444 Action [[0.70481856]] Reward -3.471291067666333 Loss 0.24707986414432526\n",
      "Episode 17 Step 3445 Action [[0.71009115]] Reward -4.113121625218863 Loss 0.19314324855804443\n",
      "Episode 17 Step 3446 Action [[0.68485143]] Reward -4.966227699185963 Loss 1.5148507356643677\n",
      "Episode 17 Step 3447 Action [[0.8648579]] Reward -6.030028939932655 Loss 0.22715607285499573\n",
      "Episode 17 Step 3448 Action [[1.05679253]] Reward -7.254067095780665 Loss 0.15430530905723572\n",
      "Episode 17 Step 3449 Action [[1.19075012]] Reward -8.579413544186531 Loss 0.08146299421787262\n",
      "Episode 17 Step 3450 Action [[1.1174347]] Reward -9.941521292513293 Loss 0.10861773788928986\n",
      "Episode 17 Step 3451 Action [[1.13346321]] Reward -11.300967055143008 Loss 0.14498524367809296\n",
      "Episode 17 Step 3452 Action [[0.8604883]] Reward -10.949587151264621 Loss 2.227156639099121\n",
      "Episode 17 Step 3453 Action [[0.86210395]] Reward -9.5641604839995 Loss 0.19611568748950958\n",
      "Episode 17 Step 3454 Action [[-0.44988546]] Reward -8.25681313670132 Loss 0.20239223539829254\n",
      "Episode 17 Step 3455 Action [[-1.14741717]] Reward -7.170963853395483 Loss 0.07087989151477814\n",
      "Episode 17 Step 3456 Action [[-1.26299101]] Reward -6.231343175129867 Loss 0.06359153985977173\n",
      "Episode 17 Step 3457 Action [[-1.12868889]] Reward -5.4017662219088 Loss 1.2041841745376587\n",
      "Episode 17 Step 3458 Action [[-1.33438616]] Reward -4.69259100713152 Loss 0.10854072868824005\n",
      "Episode 17 Step 3459 Action [[-1.18465685]] Reward -4.136583447160234 Loss 2.067095994949341\n",
      "Episode 17 Step 3460 Action [[-0.82994]] Reward -3.72440313895997 Loss 0.17634986340999603\n",
      "Episode 17 Step 3461 Action [[-1.13793842]] Reward -3.4734598595352724 Loss 0.09500262886285782\n",
      "Episode 17 Step 3462 Action [[-1.43444737]] Reward -3.396214962202673 Loss 0.09246927499771118\n",
      "Episode 17 Step 3463 Action [[-1.36871406]] Reward -3.4692770799468287 Loss 0.0961582362651825\n",
      "Episode 17 Step 3464 Action [[-1.06678978]] Reward -3.6920264366364828 Loss 0.14436228573322296\n",
      "Episode 17 Step 3465 Action [[-1.1419954]] Reward -4.089605771560164 Loss 0.16499224305152893\n",
      "Episode 17 Step 3466 Action [[-1.21869834]] Reward -4.652427835305563 Loss 0.11963629722595215\n",
      "Episode 17 Step 3467 Action [[-1.1364863]] Reward -5.370178507153399 Loss 0.0814969539642334\n",
      "Episode 17 Step 3468 Action [[-1.24750401]] Reward -6.2446405670388 Loss 0.04334516078233719\n",
      "Episode 17 Step 3469 Action [[-0.60023371]] Reward -7.230650684037681 Loss 0.045665062963962555\n",
      "Episode 17 Step 3470 Action [[0.01824805]] Reward -8.396533591620472 Loss 0.062297273427248\n",
      "Episode 17 Step 3471 Action [[1.19191318]] Reward -9.736610373050672 Loss 0.16222432255744934\n",
      "Episode 17 Step 3472 Action [[1.34112264]] Reward -11.327538378872983 Loss 0.05886353552341461\n",
      "Episode 17 Step 3473 Action [[1.0281233]] Reward -10.923368711833849 Loss 0.06416846811771393\n",
      "Episode 17 Step 3474 Action [[1.03329314]] Reward -9.643173420294694 Loss 9.268942832946777\n",
      "Episode 17 Step 3475 Action [[1.16147247]] Reward -8.353400322951384 Loss 2.9270524978637695\n",
      "Episode 17 Step 3476 Action [[1.26330603]] Reward -7.118314080605559 Loss 0.47810524702072144\n",
      "Episode 17 Step 3477 Action [[0.93071591]] Reward -5.980226892737833 Loss 0.2821351885795593\n",
      "Episode 17 Step 3478 Action [[0.58379793]] Reward -4.945719773954142 Loss 0.11722461879253387\n",
      "Episode 17 Step 3479 Action [[-0.23584189]] Reward -4.065658636712685 Loss 9.282867431640625\n",
      "Episode 17 Step 3480 Action [[0.02939194]] Reward -3.363219323968867 Loss 2.739697217941284\n",
      "Episode 17 Step 3481 Action [[0.18139737]] Reward -2.9232260772537533 Loss 0.2599460482597351\n",
      "Episode 17 Step 3482 Action [[0.76243336]] Reward -2.71807183284832 Loss 3.399512767791748\n",
      "Episode 17 Step 3483 Action [[0.97709167]] Reward -2.7178583439029316 Loss 0.25927308201789856\n",
      "Episode 17 Step 3484 Action [[1.02542892]] Reward -2.89326768644734 Loss 0.173004150390625\n",
      "Episode 17 Step 3485 Action [[0.92561233]] Reward -3.2434290501877423 Loss 0.12378789484500885\n",
      "Episode 17 Step 3486 Action [[0.85633233]] Reward -3.7851385610306045 Loss 0.3948867917060852\n",
      "Episode 17 Step 3487 Action [[0.58854838]] Reward -4.530064198273606 Loss 7.364035129547119\n",
      "Episode 17 Step 3488 Action [[0.88524055]] Reward -5.510583631103917 Loss 0.18652737140655518\n",
      "Episode 17 Step 3489 Action [[0.82609109]] Reward -6.654906864293337 Loss 0.6313989162445068\n",
      "Episode 17 Step 3490 Action [[0.79785289]] Reward -7.963314517837534 Loss 0.759790301322937\n",
      "Episode 17 Step 3491 Action [[0.80758064]] Reward -9.381875835323159 Loss 0.22701360285282135\n",
      "Episode 17 Step 3492 Action [[0.75813526]] Reward -10.836160945981815 Loss 0.2221519649028778\n",
      "Episode 17 Step 3493 Action [[0.82731767]] Reward -11.668970731471827 Loss 0.43990790843963623\n",
      "Episode 17 Step 3494 Action [[0.90635442]] Reward -10.208042448008277 Loss 0.42287367582321167\n",
      "Episode 17 Step 3495 Action [[-0.21849147]] Reward -8.791868188808197 Loss 2.7405009269714355\n",
      "Episode 17 Step 3496 Action [[-1.25128373]] Reward -7.579668604264477 Loss 0.3409225344657898\n",
      "Episode 17 Step 3497 Action [[-1.2027686]] Reward -6.543551631952084 Loss 0.4104343056678772\n",
      "Episode 17 Step 3498 Action [[-1.096599]] Reward -5.598237943100375 Loss 0.3891906142234802\n",
      "Episode 17 Step 3499 Action [[-0.80307967]] Reward -4.775303125206614 Loss 0.1267397105693817\n",
      "Episode 17 Step 3500 Action [[-0.70284692]] Reward -4.09668789425987 Loss 0.23742014169692993\n",
      "Episode 17 Step 3501 Action [[-0.90093093]] Reward -3.598303445993347 Loss 5.368986129760742\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 17 Step 3502 Action [[-1.2536509]] Reward -3.290885066321871 Loss 0.24416908621788025\n",
      "Episode 17 Step 3503 Action [[-1.11578316]] Reward -3.1546074479209887 Loss 0.11227385699748993\n",
      "Episode 17 Step 3504 Action [[-1.31812278]] Reward -3.180193296495892 Loss 2.737839698791504\n",
      "Episode 17 Step 3505 Action [[-1.37817548]] Reward -3.361973248741014 Loss 0.2475409209728241\n",
      "Episode 17 Step 3506 Action [[-1.35010868]] Reward -3.6969503619443738 Loss 0.12196957319974899\n",
      "Episode 17 Step 3507 Action [[-1.18987854]] Reward -4.190633106255358 Loss 0.11200805753469467\n",
      "Episode 17 Step 3508 Action [[-1.16761984]] Reward -4.859254446133376 Loss 0.23712190985679626\n",
      "Episode 17 Step 3509 Action [[-1.08340716]] Reward -5.693002914526793 Loss 0.08923148363828659\n",
      "Episode 17 Step 3510 Action [[-1.23857264]] Reward -6.6857319042621 Loss 0.06875497102737427\n",
      "Episode 17 Step 3511 Action [[-1.12889852]] Reward -7.777395097805296 Loss 0.08774414658546448\n",
      "Episode 17 Step 3512 Action [[0.33121318]] Reward -8.953169546542963 Loss 0.11818227171897888\n",
      "Episode 17 Step 3513 Action [[0.65964682]] Reward -10.390058490092374 Loss 0.05635439231991768\n",
      "Episode 17 Step 3514 Action [[0.77112703]] Reward -11.598200535267361 Loss 0.07545717060565948\n",
      "Episode 17 Step 3515 Action [[1.01205131]] Reward -10.347951480456848 Loss 0.10177209973335266\n",
      "Episode 17 Step 3516 Action [[0.63698118]] Reward -9.090782703450895 Loss 0.07768818736076355\n",
      "Episode 17 Step 3517 Action [[0.79741328]] Reward -7.811550641828815 Loss 0.10722102224826813\n",
      "Episode 17 Step 3518 Action [[1.20242624]] Reward -6.628067183353947 Loss 0.054399192333221436\n",
      "Episode 17 Step 3519 Action [[1.11087225]] Reward -5.594320075341169 Loss 0.06364883482456207\n",
      "Episode 17 Step 3520 Action [[0.90583393]] Reward -4.690418362520094 Loss 0.07926662266254425\n",
      "Episode 17 Step 3521 Action [[0.6267119]] Reward -3.9392856301664225 Loss 0.15236207842826843\n",
      "Episode 17 Step 3522 Action [[0.64535162]] Reward -3.3635542011578954 Loss 0.05609162896871567\n",
      "Episode 17 Step 3523 Action [[0.98462362]] Reward -2.988029558104167 Loss 0.06382044404745102\n",
      "Episode 17 Step 3524 Action [[1.39951808]] Reward -2.804436733385792 Loss 0.05095154792070389\n",
      "Episode 17 Step 3525 Action [[1.39877539]] Reward -2.7805934137203243 Loss 0.036160580813884735\n",
      "Episode 17 Step 3526 Action [[1.60314836]] Reward -2.9052122731858128 Loss 0.04117364436388016\n",
      "Episode 17 Step 3527 Action [[1.68822093]] Reward -3.1689740961386947 Loss 0.05250281095504761\n",
      "Episode 17 Step 3528 Action [[1.72484101]] Reward -3.5710606024027807 Loss 0.08195289969444275\n",
      "Episode 17 Step 3529 Action [[1.41634076]] Reward -4.113220134382601 Loss 0.09219634532928467\n",
      "Episode 17 Step 3530 Action [[1.37654002]] Reward -4.8319673842396025 Loss 0.024594662711024284\n",
      "Episode 17 Step 3531 Action [[1.5726233]] Reward -5.712201555621917 Loss 0.05897514149546623\n",
      "Episode 17 Step 3532 Action [[1.1903627]] Reward -6.704952002198156 Loss 0.0709521472454071\n",
      "Episode 17 Step 3533 Action [[1.21560859]] Reward -7.852254365493613 Loss 0.07926527410745621\n",
      "Episode 17 Step 3534 Action [[1.35505512]] Reward -9.068644112089508 Loss 9.07907485961914\n",
      "Episode 17 Step 3535 Action [[1.38677421]] Reward -10.275837420693458 Loss 0.39291560649871826\n",
      "Episode 17 Step 3536 Action [[1.28313764]] Reward -11.422237937515922 Loss 0.36914706230163574\n",
      "Episode 17 Step 3537 Action [[1.33914241]] Reward -10.131989443944361 Loss 0.2652265429496765\n",
      "Episode 17 Step 3538 Action [[1.43321592]] Reward -8.910966447643258 Loss 0.08783265948295593\n",
      "Episode 17 Step 3539 Action [[0.36165078]] Reward -7.812328372795159 Loss 0.19529220461845398\n",
      "Episode 17 Step 3540 Action [[-0.62210685]] Reward -6.93897751985867 Loss 0.21772930026054382\n",
      "Episode 17 Step 3541 Action [[-0.69075844]] Reward -6.244283492409721 Loss 0.16164125502109528\n",
      "Episode 17 Step 3542 Action [[-0.8605063]] Reward -5.6711593398403775 Loss 0.05563519522547722\n",
      "Episode 17 Step 3543 Action [[-0.65681736]] Reward -5.2339634613218875 Loss 0.16651849448680878\n",
      "Episode 17 Step 3544 Action [[-0.38150847]] Reward -4.936841587885326 Loss 0.14557692408561707\n",
      "Episode 17 Step 3545 Action [[-0.6345172]] Reward -4.806019476095983 Loss 0.04473670572042465\n",
      "Episode 17 Step 3546 Action [[-0.89108637]] Reward -4.846980969301627 Loss 0.10357378423213959\n",
      "Episode 17 Step 3547 Action [[-0.37539025]] Reward -5.0392567527366285 Loss 0.06030760332942009\n",
      "Episode 17 Step 3548 Action [[-0.50782146]] Reward -5.415917578101488 Loss 8.931136131286621\n",
      "Episode 17 Step 3549 Action [[-0.64785321]] Reward -5.955204728414375 Loss 0.22688645124435425\n",
      "Episode 17 Step 3550 Action [[-0.77133397]] Reward -6.63204861994184 Loss 0.3524470329284668\n",
      "Episode 17 Step 3551 Action [[-0.41686327]] Reward -7.41541429027311 Loss 0.3332676887512207\n",
      "Episode 17 Step 3552 Action [[0.07617557]] Reward -8.326790114910503 Loss 0.07059530913829803\n",
      "Episode 17 Step 3553 Action [[1.36970881]] Reward -9.37367232760588 Loss 0.08563243597745895\n",
      "Episode 17 Step 3554 Action [[1.66040411]] Reward -10.660024864563688 Loss 5.2154951095581055\n",
      "Episode 17 Step 3555 Action [[1.58191015]] Reward -10.535261323420322 Loss 0.0866139605641365\n",
      "Episode 17 Step 3556 Action [[1.29087966]] Reward -9.504833364215772 Loss 0.0680118054151535\n",
      "Episode 17 Step 3557 Action [[1.11060387]] Reward -8.418373468220448 Loss 0.06809644401073456\n",
      "Episode 17 Step 3558 Action [[1.20380707]] Reward -7.3369025123340395 Loss 0.15143156051635742\n",
      "Episode 17 Step 3559 Action [[0.97593379]] Reward -6.33011197268492 Loss 0.10500102490186691\n",
      "Episode 17 Step 3560 Action [[0.81465428]] Reward -5.413317542769303 Loss 0.04984237998723984\n",
      "Episode 17 Step 3561 Action [[0.70648158]] Reward -4.63384105480149 Loss 0.06970295310020447\n",
      "Episode 17 Step 3562 Action [[0.65306165]] Reward -4.022946397210286 Loss 0.04589354246854782\n",
      "Episode 17 Step 3563 Action [[0.57024294]] Reward -3.59783574578171 Loss 0.08978000283241272\n",
      "Episode 17 Step 3564 Action [[0.74210175]] Reward -3.365981128739586 Loss 0.05989671126008034\n",
      "Episode 17 Step 3565 Action [[0.80577569]] Reward -3.3275488523145462 Loss 0.04301687330007553\n",
      "Episode 17 Step 3566 Action [[0.49261102]] Reward -3.47433759377909 Loss 0.04480033740401268\n",
      "Episode 17 Step 3567 Action [[0.76263412]] Reward -3.8298238796671433 Loss 0.08456383645534515\n",
      "Episode 17 Step 3568 Action [[0.84286492]] Reward -4.373751429570223 Loss 0.03212687373161316\n",
      "Episode 17 Step 3569 Action [[0.84797251]] Reward -5.102777188260431 Loss 9.504068374633789\n",
      "Episode 17 Step 3570 Action [[0.92953397]] Reward -6.011340481054155 Loss 0.20792680978775024\n",
      "Episode 17 Step 3571 Action [[1.15501573]] Reward -7.067918487879005 Loss 0.22206133604049683\n",
      "Episode 17 Step 3572 Action [[1.03420234]] Reward -8.2075183052598 Loss 5.850173473358154\n",
      "Episode 17 Step 3573 Action [[1.36614175]] Reward -9.42161673550715 Loss 0.33902066946029663\n",
      "Episode 17 Step 3574 Action [[1.51198322]] Reward -10.585019274330795 Loss 3.6859688758850098\n",
      "Episode 17 Step 3575 Action [[1.54979932]] Reward -10.935783463163434 Loss 0.09739638864994049\n",
      "Episode 17 Step 3576 Action [[1.4118456]] Reward -9.68814043665485 Loss 0.13473141193389893\n",
      "Episode 17 Step 3577 Action [[1.02764854]] Reward -8.552797485603477 Loss 0.12225800007581711\n",
      "Episode 17 Step 3578 Action [[-0.24856576]] Reward -7.58069420897988 Loss 5.086625099182129\n",
      "Episode 17 Step 3579 Action [[-0.58452414]] Reward -6.8141166003945575 Loss 0.0680946484208107\n",
      "Episode 17 Step 3580 Action [[-0.432855]] Reward -6.173907529397666 Loss 0.1573532372713089\n",
      "Episode 17 Step 3581 Action [[-0.53895943]] Reward -5.655737696478922 Loss 0.12175555527210236\n",
      "Episode 17 Step 3582 Action [[-0.6314252]] Reward -5.287512166409011 Loss 0.08667734265327454\n",
      "Episode 17 Step 3583 Action [[-0.68928473]] Reward -5.073119049270585 Loss 0.13579104840755463\n",
      "Episode 17 Step 3584 Action [[-0.56633226]] Reward -5.013209713711486 Loss 0.1340251863002777\n",
      "Episode 17 Step 3585 Action [[-0.72164669]] Reward -5.115100810681827 Loss 0.05689913034439087\n",
      "Episode 17 Step 3586 Action [[-0.73890692]] Reward -5.370596260042442 Loss 0.076129250228405\n",
      "Episode 17 Step 3587 Action [[-0.63476143]] Reward -5.773327864847974 Loss 2.6183969974517822\n",
      "Episode 17 Step 3588 Action [[-0.80128298]] Reward -6.32364275112816 Loss 6.020901203155518\n",
      "Episode 17 Step 3589 Action [[-0.852378]] Reward -6.985244170330859 Loss 0.3772013783454895\n",
      "Episode 17 Step 3590 Action [[-0.54187497]] Reward -7.735527992435442 Loss 0.24470293521881104\n",
      "Episode 17 Step 3591 Action [[0.05627165]] Reward -8.585604987735325 Loss 0.19887247681617737\n",
      "Episode 17 Step 3592 Action [[1.09174228]] Reward -9.558699671240314 Loss 0.3155897259712219\n",
      "Episode 17 Step 3593 Action [[1.49438418]] Reward -10.712706220483929 Loss 0.21626049280166626\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 17 Step 3594 Action [[1.28302464]] Reward -10.17211695354981 Loss 0.09962458908557892\n",
      "Episode 17 Step 3595 Action [[1.38736025]] Reward -9.206646261678113 Loss 0.05012774467468262\n",
      "Episode 17 Step 3596 Action [[1.24756129]] Reward -8.23121435790168 Loss 3.478375196456909\n",
      "Episode 17 Step 3597 Action [[1.25523304]] Reward -7.263260696858031 Loss 0.20338475704193115\n",
      "Episode 17 Step 3598 Action [[1.40578762]] Reward -6.355692217629289 Loss 0.2608218789100647\n",
      "Episode 17 Step 3599 Action [[1.1985866]] Reward -5.5473741362841595 Loss 2.7212228775024414\n",
      "TOTAL REWARD @ 17-th Episode  : Reward -1265.221659969935\n",
      "Total Step: 3600\n",
      "\n",
      "Episode : 18 Replay Buffer 3600\n",
      "[-0.36657327 -0.93038918 -0.52365869]\n",
      "Episode 18 Step 3600 Action [[1.14060624]] Reward -3.816104136542612 Loss 10.830127716064453\n",
      "Episode 18 Step 3601 Action [[1.09689144]] Reward -4.106080701227921 Loss 0.20548021793365479\n",
      "Episode 18 Step 3602 Action [[1.25398905]] Reward -4.561636375250406 Loss 0.5334568023681641\n",
      "Episode 18 Step 3603 Action [[1.29433687]] Reward -5.163618598082077 Loss 0.28061068058013916\n",
      "Episode 18 Step 3604 Action [[1.35852647]] Reward -5.903699234810671 Loss 0.40081581473350525\n",
      "Episode 18 Step 3605 Action [[1.23075363]] Reward -6.757287132954793 Loss 0.1961623877286911\n",
      "Episode 18 Step 3606 Action [[1.15182662]] Reward -7.717466222626152 Loss 0.2883525490760803\n",
      "Episode 18 Step 3607 Action [[1.10027585]] Reward -8.747423399214739 Loss 0.19004309177398682\n",
      "Episode 18 Step 3608 Action [[1.10128002]] Reward -9.802270928198515 Loss 0.1624988615512848\n",
      "Episode 18 Step 3609 Action [[0.75713575]] Reward -10.828196488485537 Loss 0.30995869636535645\n",
      "Episode 18 Step 3610 Action [[0.91220848]] Reward -10.16929896101176 Loss 0.4168919026851654\n",
      "Episode 18 Step 3611 Action [[1.09299688]] Reward -9.108660491016837 Loss 0.15703850984573364\n",
      "Episode 18 Step 3612 Action [[0.59782761]] Reward -8.129371167105528 Loss 0.15047261118888855\n",
      "Episode 18 Step 3613 Action [[-0.63592539]] Reward -7.309593972767756 Loss 0.3072439730167389\n",
      "Episode 18 Step 3614 Action [[-1.00103138]] Reward -6.664202593018842 Loss 0.29030823707580566\n",
      "Episode 18 Step 3615 Action [[-1.00676359]] Reward -6.124694177478465 Loss 0.07117025554180145\n",
      "Episode 18 Step 3616 Action [[-1.33058298]] Reward -5.686222329018908 Loss 0.16579343378543854\n",
      "Episode 18 Step 3617 Action [[-1.24870897]] Reward -5.360026979917943 Loss 0.3560194671154022\n",
      "Episode 18 Step 3618 Action [[-1.26625984]] Reward -5.141601749548489 Loss 0.13367876410484314\n",
      "Episode 18 Step 3619 Action [[-1.33886284]] Reward -5.040302582379628 Loss 2.776015520095825\n",
      "Episode 18 Step 3620 Action [[-1.10077601]] Reward -5.055427107057067 Loss 0.16393797099590302\n",
      "Episode 18 Step 3621 Action [[-1.4186507]] Reward -5.20009084275786 Loss 0.13826775550842285\n",
      "Episode 18 Step 3622 Action [[-1.26436444]] Reward -5.4520954957732135 Loss 0.1758648008108139\n",
      "Episode 18 Step 3623 Action [[-1.0202972]] Reward -5.819445802867525 Loss 0.10183428227901459\n",
      "Episode 18 Step 3624 Action [[-0.9115079]] Reward -6.311453040650671 Loss 2.345252275466919\n",
      "Episode 18 Step 3625 Action [[-0.3651401]] Reward -6.916691581583155 Loss 0.09962809085845947\n",
      "Episode 18 Step 3626 Action [[0.70070241]] Reward -7.670601304612706 Loss 0.19739994406700134\n",
      "Episode 18 Step 3627 Action [[1.09995685]] Reward -8.647535686215262 Loss 0.06918954104185104\n",
      "Episode 18 Step 3628 Action [[0.85265366]] Reward -9.799422044329324 Loss 0.08259686082601547\n",
      "Episode 18 Step 3629 Action [[0.82663459]] Reward -11.010170737151766 Loss 3.539104700088501\n",
      "Episode 18 Step 3630 Action [[0.76754438]] Reward -10.064626479700637 Loss 0.05812358856201172\n",
      "Episode 18 Step 3631 Action [[0.68260233]] Reward -9.039414125764763 Loss 0.11146968603134155\n",
      "Episode 18 Step 3632 Action [[0.76500642]] Reward -8.020512492298975 Loss 0.20544208586215973\n",
      "Episode 18 Step 3633 Action [[0.83296023]] Reward -7.061610503759503 Loss 0.11483897268772125\n",
      "Episode 18 Step 3634 Action [[0.65474723]] Reward -6.194938185306822 Loss 0.028855592012405396\n",
      "Episode 18 Step 3635 Action [[0.55741358]] Reward -5.4394883062454324 Loss 0.07306849211454391\n",
      "Episode 18 Step 3636 Action [[0.30957176]] Reward -4.833717987102416 Loss 0.19424426555633545\n",
      "Episode 18 Step 3637 Action [[0.52949466]] Reward -4.400464839857318 Loss 0.10762141644954681\n",
      "Episode 18 Step 3638 Action [[0.50049819]] Reward -4.163096829540317 Loss 0.038400255143642426\n",
      "Episode 18 Step 3639 Action [[0.50703258]] Reward -4.115582998619753 Loss 0.1255236566066742\n",
      "Episode 18 Step 3640 Action [[0.48592862]] Reward -4.261139050843576 Loss 3.427273988723755\n",
      "Episode 18 Step 3641 Action [[0.56142702]] Reward -4.6020381757238695 Loss 0.12090731412172318\n",
      "Episode 18 Step 3642 Action [[0.3460885]] Reward -5.1296613033096445 Loss 0.057773251086473465\n",
      "Episode 18 Step 3643 Action [[0.16111857]] Reward -5.860297892099825 Loss 0.09936069697141647\n",
      "Episode 18 Step 3644 Action [[0.22722736]] Reward -6.792607013117329 Loss 0.1457536220550537\n",
      "Episode 18 Step 3645 Action [[0.50491766]] Reward -7.879035053121113 Loss 0.12537357211112976\n",
      "Episode 18 Step 3646 Action [[0.68444539]] Reward -9.039289160398683 Loss 0.0616418719291687\n",
      "Episode 18 Step 3647 Action [[0.94288138]] Reward -10.21606244153193 Loss 0.043447237461805344\n",
      "Episode 18 Step 3648 Action [[0.99778891]] Reward -11.111298985571278 Loss 0.07127824425697327\n",
      "Episode 18 Step 3649 Action [[0.90707806]] Reward -9.928001052979676 Loss 0.07283243536949158\n",
      "Episode 18 Step 3650 Action [[1.0052226]] Reward -8.812724748731489 Loss 0.033901285380125046\n",
      "Episode 18 Step 3651 Action [[0.98660963]] Reward -7.797718770748192 Loss 0.054001450538635254\n",
      "Episode 18 Step 3652 Action [[-0.24638819]] Reward -6.938306743258437 Loss 0.05686839669942856\n",
      "Episode 18 Step 3653 Action [[-1.10793745]] Reward -6.295605390639477 Loss 0.09170395880937576\n",
      "Episode 18 Step 3654 Action [[-0.91000191]] Reward -5.7987716034123755 Loss 0.01881716214120388\n",
      "Episode 18 Step 3655 Action [[-1.11409725]] Reward -5.408087772780099 Loss 0.06230950728058815\n",
      "Episode 18 Step 3656 Action [[-1.29343335]] Reward -5.144512670511629 Loss 0.045360371470451355\n",
      "Episode 18 Step 3657 Action [[-1.54368277]] Reward -5.00258706600311 Loss 0.026728348806500435\n",
      "Episode 18 Step 3658 Action [[-1.18308994]] Reward -4.970233353036121 Loss 0.026449445635080338\n",
      "Episode 18 Step 3659 Action [[-1.06599345]] Reward -5.061201773818822 Loss 0.05162167549133301\n",
      "Episode 18 Step 3660 Action [[-0.79402712]] Reward -5.285325648341357 Loss 0.0659806877374649\n",
      "Episode 18 Step 3661 Action [[-0.77995582]] Reward -5.658973276260506 Loss 0.023915991187095642\n",
      "Episode 18 Step 3662 Action [[-0.49810615]] Reward -6.17132699768623 Loss 0.025953875854611397\n",
      "Episode 18 Step 3663 Action [[0.084557]] Reward -6.835990948121366 Loss 0.041163165122270584\n",
      "Episode 18 Step 3664 Action [[0.78181955]] Reward -7.687378642608519 Loss 0.019108464941382408\n",
      "Episode 18 Step 3665 Action [[2.04709788]] Reward -8.756752473788419 Loss 0.014347918331623077\n",
      "Episode 18 Step 3666 Action [[1.70654672]] Reward -10.127760405129976 Loss 0.01680693030357361\n",
      "Episode 18 Step 3667 Action [[1.64317093]] Reward -11.171559780225847 Loss 0.020851997658610344\n",
      "Episode 18 Step 3668 Action [[1.47896154]] Reward -10.112282061803427 Loss 0.026124488562345505\n",
      "Episode 18 Step 3669 Action [[1.43968282]] Reward -8.985605649789704 Loss 0.01954847387969494\n",
      "Episode 18 Step 3670 Action [[1.5883655]] Reward -7.847669064980311 Loss 0.012215901166200638\n",
      "Episode 18 Step 3671 Action [[1.72943679]] Reward -6.760410778266498 Loss 7.923552513122559\n",
      "Episode 18 Step 3672 Action [[1.44608141]] Reward -5.757755036929335 Loss 0.21170389652252197\n",
      "Episode 18 Step 3673 Action [[1.0044398]] Reward -4.840656648767363 Loss 0.30644458532333374\n",
      "Episode 18 Step 3674 Action [[0.73250735]] Reward -4.043998883886178 Loss 0.14083847403526306\n",
      "Episode 18 Step 3675 Action [[0.30983828]] Reward -3.4159809763641436 Loss 0.057856570929288864\n",
      "Episode 18 Step 3676 Action [[0.49563754]] Reward -2.977256748538874 Loss 8.410017013549805\n",
      "Episode 18 Step 3677 Action [[1.11393733]] Reward -2.755334674265179 Loss 0.12542815506458282\n",
      "Episode 18 Step 3678 Action [[1.3609807]] Reward -2.719152787249456 Loss 0.2978866398334503\n",
      "Episode 18 Step 3679 Action [[1.32057286]] Reward -2.8359104113330917 Loss 0.1965387463569641\n",
      "Episode 18 Step 3680 Action [[0.98718827]] Reward -3.1074161311876036 Loss 0.22569115459918976\n",
      "Episode 18 Step 3681 Action [[1.17288732]] Reward -3.5672488501370694 Loss 0.12575066089630127\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 18 Step 3682 Action [[1.08549789]] Reward -4.198077688303191 Loss 0.1617409586906433\n",
      "Episode 18 Step 3683 Action [[1.36092762]] Reward -5.020051831121194 Loss 0.19353607296943665\n",
      "Episode 18 Step 3684 Action [[1.61556125]] Reward -5.985262456167581 Loss 0.07464290410280228\n",
      "Episode 18 Step 3685 Action [[1.70623215]] Reward -7.0532053093621 Loss 0.11590632051229477\n",
      "Episode 18 Step 3686 Action [[1.43758679]] Reward -8.192726258670064 Loss 0.09934338927268982\n",
      "Episode 18 Step 3687 Action [[1.07037363]] Reward -9.406635993379474 Loss 0.13717927038669586\n",
      "Episode 18 Step 3688 Action [[1.33232696]] Reward -10.670661752707623 Loss 0.09041211009025574\n",
      "Episode 18 Step 3689 Action [[1.19880895]] Reward -11.118591243371915 Loss 0.12224578857421875\n",
      "Episode 18 Step 3690 Action [[1.09694554]] Reward -9.814587245062194 Loss 0.1405714750289917\n",
      "Episode 18 Step 3691 Action [[1.39867081]] Reward -8.600072852541867 Loss 0.08712571859359741\n",
      "Episode 18 Step 3692 Action [[0.21551247]] Reward -7.497625790297233 Loss 0.04599478095769882\n",
      "Episode 18 Step 3693 Action [[-0.42916997]] Reward -6.6381228463357225 Loss 0.0750652328133583\n",
      "Episode 18 Step 3694 Action [[-0.5996094]] Reward -5.9521982967439895 Loss 0.12452574819326401\n",
      "Episode 18 Step 3695 Action [[-0.59146062]] Reward -5.411217160546576 Loss 0.05915214493870735\n",
      "Episode 18 Step 3696 Action [[-0.50550365]] Reward -5.019854425789839 Loss 0.060319408774375916\n",
      "Episode 18 Step 3697 Action [[-0.25785753]] Reward -4.791072258270711 Loss 0.06948328763246536\n",
      "Episode 18 Step 3698 Action [[-0.45832558]] Reward -4.74380959235339 Loss 0.07444064319133759\n",
      "Episode 18 Step 3699 Action [[-0.84271174]] Reward -4.880136821972298 Loss 0.07987711578607559\n",
      "Episode 18 Step 3700 Action [[-1.10650479]] Reward -5.170884909786343 Loss 0.05709071829915047\n",
      "Episode 18 Step 3701 Action [[-0.80459924]] Reward -5.589700971073125 Loss 0.07186219096183777\n",
      "Episode 18 Step 3702 Action [[-0.701146]] Reward -6.160046173839829 Loss 3.0081260204315186\n",
      "Episode 18 Step 3703 Action [[-0.98702428]] Reward -6.868295028825434 Loss 0.08948007225990295\n",
      "Episode 18 Step 3704 Action [[-1.09862613]] Reward -7.652203022127596 Loss 0.06379468739032745\n",
      "Episode 18 Step 3705 Action [[-0.78560248]] Reward -8.482618965675318 Loss 0.09494104236364365\n",
      "Episode 18 Step 3706 Action [[-0.0900367]] Reward -9.372808672414127 Loss 0.08575847744941711\n",
      "Episode 18 Step 3707 Action [[0.57001383]] Reward -10.354703354003579 Loss 0.12051020562648773\n",
      "Episode 18 Step 3708 Action [[0.41246773]] Reward -10.282781967696689 Loss 11.766878128051758\n",
      "Episode 18 Step 3709 Action [[0.63069674]] Reward -9.353197597567991 Loss 0.5066548585891724\n",
      "Episode 18 Step 3710 Action [[0.50349963]] Reward -8.445466612884468 Loss 0.9869712591171265\n",
      "Episode 18 Step 3711 Action [[0.61027855]] Reward -7.568375604229544 Loss 0.5327199697494507\n",
      "Episode 18 Step 3712 Action [[0.94575312]] Reward -6.772903181875461 Loss 0.40040335059165955\n",
      "Episode 18 Step 3713 Action [[0.97489529]] Reward -6.089279842151766 Loss 0.42094624042510986\n",
      "Episode 18 Step 3714 Action [[0.87336188]] Reward -5.514512066412508 Loss 0.5122337937355042\n",
      "Episode 18 Step 3715 Action [[0.95797121]] Reward -5.064411135431707 Loss 0.14497385919094086\n",
      "Episode 18 Step 3716 Action [[0.953894]] Reward -4.757501323667997 Loss 0.2385714203119278\n",
      "Episode 18 Step 3717 Action [[0.59379606]] Reward -4.5967185693060655 Loss 0.23557451367378235\n",
      "Episode 18 Step 3718 Action [[0.73431229]] Reward -4.600284632014838 Loss 0.540017306804657\n",
      "Episode 18 Step 3719 Action [[0.52842323]] Reward -4.771888035537403 Loss 0.2956547737121582\n",
      "Episode 18 Step 3720 Action [[0.87377604]] Reward -5.123707306809037 Loss 0.28312018513679504\n",
      "Episode 18 Step 3721 Action [[1.01450273]] Reward -5.620118342818567 Loss 0.22790303826332092\n",
      "Episode 18 Step 3722 Action [[1.10220306]] Reward -6.243342704193139 Loss 0.1914379894733429\n",
      "Episode 18 Step 3723 Action [[1.4407499]] Reward -6.9715536404944185 Loss 0.12083195149898529\n",
      "Episode 18 Step 3724 Action [[1.25592083]] Reward -7.741674253466573 Loss 0.11201581358909607\n",
      "Episode 18 Step 3725 Action [[1.2441323]] Reward -8.568406381863701 Loss 0.137879878282547\n",
      "Episode 18 Step 3726 Action [[1.18783821]] Reward -9.40460516732199 Loss 0.27004820108413696\n",
      "Episode 18 Step 3727 Action [[0.84996324]] Reward -10.220344016226917 Loss 0.22757717967033386\n",
      "Episode 18 Step 3728 Action [[0.88844599]] Reward -10.170698749909675 Loss 0.21071378886699677\n",
      "Episode 18 Step 3729 Action [[0.77222397]] Reward -9.32164914272502 Loss 0.21817542612552643\n",
      "Episode 18 Step 3730 Action [[0.64991648]] Reward -8.549708949108918 Loss 0.1318640410900116\n",
      "Episode 18 Step 3731 Action [[-0.52152]] Reward -7.879955168685299 Loss 0.0775376483798027\n",
      "Episode 18 Step 3732 Action [[-0.95471068]] Reward -7.342804386540463 Loss 0.08735664188861847\n",
      "Episode 18 Step 3733 Action [[-0.99274597]] Reward -6.884361400980388 Loss 0.11454634368419647\n",
      "Episode 18 Step 3734 Action [[-0.60929827]] Reward -6.496980412998661 Loss 0.20705583691596985\n",
      "Episode 18 Step 3735 Action [[-1.1598279]] Reward -6.1985885805397904 Loss 0.1298590898513794\n",
      "Episode 18 Step 3736 Action [[-1.29899225]] Reward -6.004957841338422 Loss 0.1258019655942917\n",
      "Episode 18 Step 3737 Action [[-1.49822812]] Reward -5.900148216696437 Loss 0.05051402747631073\n",
      "Episode 18 Step 3738 Action [[-1.24089924]] Reward -5.875897350689994 Loss 0.06101691350340843\n",
      "Episode 18 Step 3739 Action [[-1.47114434]] Reward -5.943734942817115 Loss 0.03287183493375778\n",
      "Episode 18 Step 3740 Action [[-1.36982719]] Reward -6.090577216068848 Loss 0.08344465494155884\n",
      "Episode 18 Step 3741 Action [[-1.21778555]] Reward -6.319111009839596 Loss 0.11171829700469971\n",
      "Episode 18 Step 3742 Action [[-0.77134494]] Reward -6.631873040287066 Loss 0.09140138328075409\n",
      "Episode 18 Step 3743 Action [[-0.8070849]] Reward -7.052920595745001 Loss 0.12529727816581726\n",
      "Episode 18 Step 3744 Action [[-0.29829156]] Reward -7.552643289404842 Loss 0.08657132089138031\n",
      "Episode 18 Step 3745 Action [[0.13062799]] Reward -8.163643757581006 Loss 0.05567967891693115\n",
      "Episode 18 Step 3746 Action [[0.88996615]] Reward -8.887403465577496 Loss 0.05537980794906616\n",
      "Episode 18 Step 3747 Action [[1.4758824]] Reward -9.761793140773793 Loss 0.13915495574474335\n",
      "Episode 18 Step 3748 Action [[1.40916645]] Reward -10.49830066427449 Loss 0.09519001841545105\n",
      "Episode 18 Step 3749 Action [[1.40509417]] Reward -9.707257965415963 Loss 3.971147060394287\n",
      "Episode 18 Step 3750 Action [[1.87612727]] Reward -8.876763153311723 Loss 0.07442263513803482\n",
      "Episode 18 Step 3751 Action [[1.54537378]] Reward -8.05493967693346 Loss 0.08775587379932404\n",
      "Episode 18 Step 3752 Action [[1.23339942]] Reward -7.214380365186534 Loss 0.07618285715579987\n",
      "Episode 18 Step 3753 Action [[1.37803413]] Reward -6.403141585987562 Loss 0.0545242540538311\n",
      "Episode 18 Step 3754 Action [[1.23430838]] Reward -5.683609475796832 Loss 0.07580322027206421\n",
      "Episode 18 Step 3755 Action [[1.17383639]] Reward -5.063304735306479 Loss 0.13832210004329681\n",
      "Episode 18 Step 3756 Action [[0.72919967]] Reward -4.56925162807452 Loss 0.07139050960540771\n",
      "Episode 18 Step 3757 Action [[0.73416559]] Reward -4.217668875616891 Loss 0.06587115675210953\n",
      "Episode 18 Step 3758 Action [[0.71552602]] Reward -4.041325543007848 Loss 0.060507096350193024\n",
      "Episode 18 Step 3759 Action [[0.44740053]] Reward -4.043746261476011 Loss 0.06850293278694153\n",
      "Episode 18 Step 3760 Action [[0.66392204]] Reward -4.242124197080479 Loss 0.0813908576965332\n",
      "Episode 18 Step 3761 Action [[0.61712197]] Reward -4.624852017032307 Loss 0.07326770573854446\n",
      "Episode 18 Step 3762 Action [[0.60813166]] Reward -5.194697850255081 Loss 0.17833945155143738\n",
      "Episode 18 Step 3763 Action [[0.25786999]] Reward -5.942886806655495 Loss 0.10559666156768799\n",
      "Episode 18 Step 3764 Action [[0.34130179]] Reward -6.894790058983874 Loss 0.07035670429468155\n",
      "Episode 18 Step 3765 Action [[0.54622251]] Reward -7.988795564302115 Loss 0.030674610286951065\n",
      "Episode 18 Step 3766 Action [[0.53042109]] Reward -9.155255340130639 Loss 0.02228725142776966\n",
      "Episode 18 Step 3767 Action [[0.90877639]] Reward -10.361439636957376 Loss 0.03979850932955742\n",
      "Episode 18 Step 3768 Action [[1.13445641]] Reward -11.010709649989142 Loss 0.043800532817840576\n",
      "Episode 18 Step 3769 Action [[1.27357096]] Reward -9.808368261138044 Loss 0.03180058300495148\n",
      "Episode 18 Step 3770 Action [[1.0144063]] Reward -8.673291441752118 Loss 0.04659339413046837\n",
      "Episode 18 Step 3771 Action [[0.63678531]] Reward -7.6826456894515855 Loss 6.777990341186523\n",
      "Episode 18 Step 3772 Action [[-0.38053258]] Reward -6.864818559855664 Loss 0.24083212018013\n",
      "Episode 18 Step 3773 Action [[-1.00184363]] Reward -6.238193365469431 Loss 0.3106992840766907\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 18 Step 3774 Action [[-1.2366663]] Reward -5.749757938981467 Loss 0.24296250939369202\n",
      "Episode 18 Step 3775 Action [[-1.58044581]] Reward -5.375289456782778 Loss 0.14476537704467773\n",
      "Episode 18 Step 3776 Action [[-1.54081312]] Reward -5.108580017178868 Loss 4.326381206512451\n",
      "Episode 18 Step 3777 Action [[-1.56312529]] Reward -4.943232461072883 Loss 0.19918286800384521\n",
      "Episode 18 Step 3778 Action [[-1.40706086]] Reward -4.883927819906877 Loss 12.073634147644043\n",
      "Episode 18 Step 3779 Action [[-1.29115198]] Reward -4.939180115765786 Loss 5.702359676361084\n",
      "Episode 18 Step 3780 Action [[-0.90160359]] Reward -5.116795963894009 Loss 1.9825866222381592\n",
      "Episode 18 Step 3781 Action [[-1.00831017]] Reward -5.441979753402749 Loss 1.9380922317504883\n",
      "Episode 18 Step 3782 Action [[-0.92166912]] Reward -5.895710572998184 Loss 0.6156119704246521\n",
      "Episode 18 Step 3783 Action [[-0.84109029]] Reward -6.476786212778654 Loss 0.9126023054122925\n",
      "Episode 18 Step 3784 Action [[-0.82748974]] Reward -7.1728322077618545 Loss 1.1315267086029053\n",
      "Episode 18 Step 3785 Action [[0.09603193]] Reward -7.955798716874686 Loss 7.689591884613037\n",
      "Episode 18 Step 3786 Action [[1.03755135]] Reward -8.910886436246741 Loss 0.8951386213302612\n",
      "Episode 18 Step 3787 Action [[1.30970815]] Reward -10.064980479014176 Loss 12.804481506347656\n",
      "Episode 18 Step 3788 Action [[1.21728963]] Reward -10.822865035029846 Loss 3.551875591278076\n",
      "Episode 18 Step 3789 Action [[0.99926738]] Reward -9.838692455347317 Loss 1.7094746828079224\n",
      "Episode 18 Step 3790 Action [[1.34075855]] Reward -8.806747297711514 Loss 0.32462191581726074\n",
      "Episode 18 Step 3791 Action [[1.0969872]] Reward -7.807336508750312 Loss 2.5625839233398438\n",
      "Episode 18 Step 3792 Action [[1.12252364]] Reward -6.828651845971987 Loss 1.7637962102890015\n",
      "Episode 18 Step 3793 Action [[1.25951549]] Reward -5.936426769682039 Loss 0.1203121691942215\n",
      "Episode 18 Step 3794 Action [[1.32514103]] Reward -5.16511300749635 Loss 0.5026668310165405\n",
      "Episode 18 Step 3795 Action [[1.00111787]] Reward -4.525003846405077 Loss 1.0806097984313965\n",
      "Episode 18 Step 3796 Action [[1.33122708]] Reward -4.020276040352358 Loss 0.8582789897918701\n",
      "Episode 18 Step 3797 Action [[1.13310471]] Reward -3.681059572800853 Loss 0.46024206280708313\n",
      "Episode 18 Step 3798 Action [[1.51860322]] Reward -3.4944584687124833 Loss 0.6569955348968506\n",
      "Episode 18 Step 3799 Action [[1.51897727]] Reward -3.457983067431326 Loss 1.9426259994506836\n",
      "Now we save model\n",
      "TOTAL REWARD @ 18-th Episode  : Reward -1353.687675907963\n",
      "Total Step: 3800\n",
      "\n",
      "Episode : 19 Replay Buffer 3800\n",
      "[-0.49375773  0.86959951  0.0548933 ]\n",
      "Episode 19 Step 3800 Action [[-0.22269648]] Reward -4.356763227883017 Loss 2.9175896644592285\n",
      "Episode 19 Step 3801 Action [[-0.25993879]] Reward -4.543612521708374 Loss 0.5050046443939209\n",
      "Episode 19 Step 3802 Action [[-0.6255129]] Reward -4.935143191196444 Loss 2.1001510620117188\n",
      "Episode 19 Step 3803 Action [[-0.74156221]] Reward -5.494392749640732 Loss 0.5125679969787598\n",
      "Episode 19 Step 3804 Action [[-0.77358026]] Reward -6.206244529467849 Loss 0.14344006776809692\n",
      "Episode 19 Step 3805 Action [[-0.69451491]] Reward -7.051797495075306 Loss 0.5616142749786377\n",
      "Episode 19 Step 3806 Action [[-0.36692227]] Reward -8.013317403975993 Loss 1.6261180639266968\n",
      "Episode 19 Step 3807 Action [[0.51475283]] Reward -9.092424110743089 Loss 0.1558907926082611\n",
      "Episode 19 Step 3808 Action [[1.22888121]] Reward -10.34784851361188 Loss 0.2671358585357666\n",
      "Episode 19 Step 3809 Action [[1.22989779]] Reward -10.971127841423053 Loss 1.1149325370788574\n",
      "Episode 19 Step 3810 Action [[1.26360386]] Reward -9.888801019187369 Loss 0.37223923206329346\n",
      "Episode 19 Step 3811 Action [[1.01968541]] Reward -8.774944434208566 Loss 0.22350826859474182\n",
      "Episode 19 Step 3812 Action [[0.83280938]] Reward -7.648526420602062 Loss 3.494405508041382\n",
      "Episode 19 Step 3813 Action [[1.08232791]] Reward -6.572303976097782 Loss 0.350760817527771\n",
      "Episode 19 Step 3814 Action [[1.1295378]] Reward -5.62507135757166 Loss 4.253328323364258\n",
      "Episode 19 Step 3815 Action [[1.27775936]] Reward -4.8132424776137634 Loss 10.073498725891113\n",
      "Episode 19 Step 3816 Action [[0.88525622]] Reward -4.1561779163116945 Loss 1.0837860107421875\n",
      "Episode 19 Step 3817 Action [[1.14198358]] Reward -3.644926999917224 Loss 2.18768048286438\n",
      "Episode 19 Step 3818 Action [[1.07458653]] Reward -3.313008818349794 Loss 0.8492348790168762\n",
      "Episode 19 Step 3819 Action [[1.41206938]] Reward -3.1464177224211207 Loss 0.4878636598587036\n",
      "Episode 19 Step 3820 Action [[1.38341954]] Reward -3.136372157049442 Loss 0.550483763217926\n",
      "Episode 19 Step 3821 Action [[1.3298398]] Reward -3.275861428611401 Loss 0.562111496925354\n",
      "Episode 19 Step 3822 Action [[1.1971602]] Reward -3.571148769093339 Loss 0.4819812476634979\n",
      "Episode 19 Step 3823 Action [[1.48905416]] Reward -4.037269444642919 Loss 0.640239953994751\n",
      "Episode 19 Step 3824 Action [[1.54806182]] Reward -4.644165538350575 Loss 0.8495841026306152\n",
      "Episode 19 Step 3825 Action [[1.14074424]] Reward -5.390495359081117 Loss 0.4679507613182068\n",
      "Episode 19 Step 3826 Action [[1.01066752]] Reward -6.318813557753209 Loss 0.24157719314098358\n",
      "Episode 19 Step 3827 Action [[1.14086747]] Reward -7.395841936920113 Loss 0.15593209862709045\n",
      "Episode 19 Step 3828 Action [[1.35410402]] Reward -8.551420467964586 Loss 0.2708790600299835\n",
      "Episode 19 Step 3829 Action [[1.55786651]] Reward -9.711718290942148 Loss 0.5313057899475098\n",
      "Episode 19 Step 3830 Action [[1.35812638]] Reward -10.8097864873929 Loss 0.24569784104824066\n",
      "Episode 19 Step 3831 Action [[1.53539973]] Reward -10.571156609381404 Loss 0.18867072463035583\n",
      "Episode 19 Step 3832 Action [[1.23172955]] Reward -9.368744415018872 Loss 0.12428378313779831\n",
      "Episode 19 Step 3833 Action [[0.38403542]] Reward -8.29832022634322 Loss 0.13879255950450897\n",
      "Episode 19 Step 3834 Action [[-1.1173384]] Reward -7.408782202562545 Loss 0.09456655383110046\n",
      "Episode 19 Step 3835 Action [[-1.26717957]] Reward -6.697197397511411 Loss 0.16328030824661255\n",
      "Episode 19 Step 3836 Action [[-0.98141413]] Reward -6.065627786730324 Loss 0.12179463356733322\n",
      "Episode 19 Step 3837 Action [[-0.71905819]] Reward -5.520237179421607 Loss 0.10849131643772125\n",
      "Episode 19 Step 3838 Action [[-0.73692294]] Reward -5.0977989122577245 Loss 2.7079548835754395\n",
      "Episode 19 Step 3839 Action [[-0.75066369]] Reward -4.8266300193534715 Loss 0.044511646032333374\n",
      "Episode 19 Step 3840 Action [[-0.75856219]] Reward -4.712948790353156 Loss 0.18453764915466309\n",
      "Episode 19 Step 3841 Action [[-0.82752436]] Reward -4.759891198704061 Loss 0.17597275972366333\n",
      "Episode 19 Step 3842 Action [[-0.86683585]] Reward -4.964217147431234 Loss 0.13955646753311157\n",
      "Episode 19 Step 3843 Action [[-0.83861779]] Reward -5.320175931832752 Loss 0.07465416938066483\n",
      "Episode 19 Step 3844 Action [[-1.31187719]] Reward -5.825015263826886 Loss 0.15915091335773468\n",
      "Episode 19 Step 3845 Action [[-1.45428123]] Reward -6.418239256306901 Loss 0.11108380556106567\n",
      "Episode 19 Step 3846 Action [[-1.56945696]] Reward -7.084334364664973 Loss 0.05373353138566017\n",
      "Episode 19 Step 3847 Action [[-1.24268147]] Reward -7.7920857984375465 Loss 0.044635869562625885\n",
      "Episode 19 Step 3848 Action [[-0.81831402]] Reward -8.559913146300243 Loss 0.1098291426897049\n",
      "Episode 19 Step 3849 Action [[-0.19502894]] Reward -9.386387134644878 Loss 0.09403207153081894\n",
      "Episode 19 Step 3850 Action [[0.19095747]] Reward -10.285996736969736 Loss 0.10688023269176483\n",
      "Episode 19 Step 3851 Action [[0.61962659]] Reward -10.14907711320246 Loss 0.042707592248916626\n",
      "Episode 19 Step 3852 Action [[0.46333453]] Reward -9.30784511605113 Loss 0.12359270453453064\n",
      "Episode 19 Step 3853 Action [[0.4451584]] Reward -8.462534926023164 Loss 0.1146615520119667\n",
      "Episode 19 Step 3854 Action [[0.60033553]] Reward -7.657985496985987 Loss 0.04545006901025772\n",
      "Episode 19 Step 3855 Action [[0.5369561]] Reward -6.931899669844881 Loss 0.05453729256987572\n",
      "Episode 19 Step 3856 Action [[0.68979535]] Reward -6.298628790774448 Loss 0.07460923492908478\n",
      "Episode 19 Step 3857 Action [[0.73207148]] Reward -5.787980088259498 Loss 0.12398035824298859\n",
      "Episode 19 Step 3858 Action [[0.73456521]] Reward -5.407038339147608 Loss 0.03146457299590111\n",
      "Episode 19 Step 3859 Action [[0.66953579]] Reward -5.165441267901835 Loss 3.048847198486328\n",
      "Episode 19 Step 3860 Action [[0.73181903]] Reward -5.072735914122122 Loss 0.05556092411279678\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 19 Step 3861 Action [[0.899073]] Reward -5.132407546915585 Loss 0.04473695158958435\n",
      "Episode 19 Step 3862 Action [[0.83984866]] Reward -5.333521884188896 Loss 0.0933406874537468\n",
      "Episode 19 Step 3863 Action [[0.61396938]] Reward -5.675812058105432 Loss 0.04254905879497528\n",
      "Episode 19 Step 3864 Action [[0.33955694]] Reward -6.1700170220894535 Loss 0.046491943299770355\n",
      "Episode 19 Step 3865 Action [[0.04430961]] Reward -6.82456024596285 Loss 3.5743134021759033\n",
      "Episode 19 Step 3866 Action [[0.33360842]] Reward -7.641642660159259 Loss 0.02331915870308876\n",
      "Episode 19 Step 3867 Action [[0.09585428]] Reward -8.53956847330233 Loss 0.03586633875966072\n",
      "Episode 19 Step 3868 Action [[0.40612348]] Reward -9.529514324076205 Loss 0.039620399475097656\n",
      "Episode 19 Step 3869 Action [[0.25426372]] Reward -10.506179448778749 Loss 0.069765105843544\n",
      "Episode 19 Step 3870 Action [[0.45702741]] Reward -10.157251215751115 Loss 0.0717591643333435\n",
      "Episode 19 Step 3871 Action [[0.1808624]] Reward -9.206574445372702 Loss 0.030249066650867462\n",
      "Episode 19 Step 3872 Action [[-0.71868065]] Reward -8.325821176977957 Loss 0.06582662463188171\n",
      "Episode 19 Step 3873 Action [[-0.91544021]] Reward -7.5555478824002655 Loss 0.03604291379451752\n",
      "Episode 19 Step 3874 Action [[-1.39880659]] Reward -6.855763755580526 Loss 0.030710561200976372\n",
      "Episode 19 Step 3875 Action [[-1.3826989]] Reward -6.246987024901193 Loss 0.04959595203399658\n",
      "Episode 19 Step 3876 Action [[-1.47080007]] Reward -5.715480188152827 Loss 0.07066424190998077\n",
      "Episode 19 Step 3877 Action [[-1.54233799]] Reward -5.279983829151004 Loss 0.0471280962228775\n",
      "Episode 19 Step 3878 Action [[-1.31566582]] Reward -4.946828646920935 Loss 6.105874538421631\n",
      "Episode 19 Step 3879 Action [[-1.45467236]] Reward -4.72535196759184 Loss 0.09235633909702301\n",
      "Episode 19 Step 3880 Action [[-1.53924834]] Reward -4.626571975207135 Loss 0.06921741366386414\n",
      "Episode 19 Step 3881 Action [[-1.31165133]] Reward -4.6450169730859 Loss 0.10451851785182953\n",
      "Episode 19 Step 3882 Action [[-1.34735419]] Reward -4.792384486418365 Loss 0.07488012313842773\n",
      "Episode 19 Step 3883 Action [[-0.94666911]] Reward -5.065266910043182 Loss 0.08766906708478928\n",
      "Episode 19 Step 3884 Action [[-0.74985211]] Reward -5.491320585111328 Loss 0.09526412189006805\n",
      "Episode 19 Step 3885 Action [[-0.63718271]] Reward -6.072946748056527 Loss 0.03445345163345337\n",
      "Episode 19 Step 3886 Action [[-0.55823075]] Reward -6.800731294860735 Loss 0.0513068251311779\n",
      "Episode 19 Step 3887 Action [[-0.7547566]] Reward -7.655411151193626 Loss 0.027675466611981392\n",
      "Episode 19 Step 3888 Action [[-0.78677911]] Reward -8.573415735711189 Loss 0.040518537163734436\n",
      "Episode 19 Step 3889 Action [[-0.20777394]] Reward -9.523944182257782 Loss 10.669922828674316\n",
      "Episode 19 Step 3890 Action [[0.54246686]] Reward -10.549555134377458 Loss 0.22681160271167755\n",
      "Episode 19 Step 3891 Action [[1.11617245]] Reward -10.240741061128565 Loss 0.189418226480484\n",
      "Episode 19 Step 3892 Action [[1.14818533]] Reward -9.308550942110395 Loss 0.1613341122865677\n",
      "Episode 19 Step 3893 Action [[0.92536124]] Reward -8.363105099864079 Loss 6.031598091125488\n",
      "Episode 19 Step 3894 Action [[0.93536758]] Reward -7.424440863995172 Loss 0.13774287700653076\n",
      "Episode 19 Step 3895 Action [[1.10933854]] Reward -6.552201454988157 Loss 0.11526121199131012\n",
      "Episode 19 Step 3896 Action [[0.95479078]] Reward -5.784542056701739 Loss 0.189848855137825\n",
      "Episode 19 Step 3897 Action [[0.85618266]] Reward -5.1273532332591625 Loss 0.150197833776474\n",
      "Episode 19 Step 3898 Action [[0.82517342]] Reward -4.61108100659413 Loss 0.23761694133281708\n",
      "Episode 19 Step 3899 Action [[0.52701116]] Reward -4.254471056173962 Loss 0.3496350049972534\n",
      "Episode 19 Step 3900 Action [[0.56064298]] Reward -4.071509993979289 Loss 3.905809164047241\n",
      "Episode 19 Step 3901 Action [[0.6274723]] Reward -4.078714883507757 Loss 0.1452786922454834\n",
      "Episode 19 Step 3902 Action [[0.60470648]] Reward -4.273398078231417 Loss 0.14953060448169708\n",
      "Episode 19 Step 3903 Action [[0.44492735]] Reward -4.656598072485035 Loss 0.23919302225112915\n",
      "Episode 19 Step 3904 Action [[0.54371558]] Reward -5.240109852062735 Loss 0.1401294320821762\n",
      "Episode 19 Step 3905 Action [[0.64143809]] Reward -6.001144135732969 Loss 0.13675998151302338\n",
      "Episode 19 Step 3906 Action [[0.60437588]] Reward -6.913037649631939 Loss 0.09071405231952667\n",
      "Episode 19 Step 3907 Action [[0.71958084]] Reward -7.956275716941335 Loss 0.06918986141681671\n",
      "Episode 19 Step 3908 Action [[0.55380457]] Reward -9.068227741770524 Loss 3.736204147338867\n",
      "Episode 19 Step 3909 Action [[0.64651247]] Reward -10.234096439760348 Loss 0.13996991515159607\n",
      "Episode 19 Step 3910 Action [[0.65625069]] Reward -11.026033673523296 Loss 0.17376762628555298\n",
      "Episode 19 Step 3911 Action [[0.43911902]] Reward -9.874356598426564 Loss 10.066268920898438\n",
      "Episode 19 Step 3912 Action [[0.42562103]] Reward -8.781861665495937 Loss 0.32313477993011475\n",
      "Episode 19 Step 3913 Action [[-1.14434176]] Reward -7.771251950687741 Loss 0.3343091607093811\n",
      "Episode 19 Step 3914 Action [[-1.66632344]] Reward -6.949888132411145 Loss 0.16714438796043396\n",
      "Episode 19 Step 3915 Action [[-1.49625727]] Reward -6.217476075019155 Loss 0.22377422451972961\n",
      "Episode 19 Step 3916 Action [[-1.56703174]] Reward -5.5504604641790625 Loss 0.22813041508197784\n",
      "Episode 19 Step 3917 Action [[-1.70636207]] Reward -4.985280431392218 Loss 9.680665016174316\n",
      "Episode 19 Step 3918 Action [[-1.46679523]] Reward -4.532022196744547 Loss 0.18284326791763306\n",
      "Episode 19 Step 3919 Action [[-1.44967277]] Reward -4.1919539675998365 Loss 0.37205296754837036\n",
      "Episode 19 Step 3920 Action [[-1.52187659]] Reward -3.9843252820453925 Loss 6.362708568572998\n",
      "Episode 19 Step 3921 Action [[-1.56802469]] Reward -3.910874591260515 Loss 0.15661367774009705\n",
      "Episode 19 Step 3922 Action [[-1.59368939]] Reward -3.9691155485176823 Loss 0.12122628837823868\n",
      "Episode 19 Step 3923 Action [[-1.39404628]] Reward -4.156993911633489 Loss 0.21527838706970215\n",
      "Episode 19 Step 3924 Action [[-1.51975289]] Reward -4.4888953688253626 Loss 0.3238440752029419\n",
      "Episode 19 Step 3925 Action [[-1.41172821]] Reward -4.9490161821659475 Loss 0.10620670020580292\n",
      "Episode 19 Step 3926 Action [[-1.37630648]] Reward -5.544536284292949 Loss 0.1765611171722412\n",
      "Episode 19 Step 3927 Action [[-1.268858]] Reward -6.262231941260776 Loss 0.10075651109218597\n",
      "Episode 19 Step 3928 Action [[-1.54921563]] Reward -7.093553526696996 Loss 0.18094919621944427\n",
      "Episode 19 Step 3929 Action [[-1.34796929]] Reward -7.963782626912823 Loss 0.18367567658424377\n",
      "Episode 19 Step 3930 Action [[-0.64278744]] Reward -8.884644818495522 Loss 0.16799868643283844\n",
      "Episode 19 Step 3931 Action [[0.02081033]] Reward -9.899237859309938 Loss 0.09988468885421753\n",
      "Episode 19 Step 3932 Action [[0.47901629]] Reward -10.94794943866555 Loss 0.13634052872657776\n",
      "Episode 19 Step 3933 Action [[0.58354882]] Reward -9.95942144126288 Loss 0.04825182259082794\n",
      "Episode 19 Step 3934 Action [[0.40782133]] Reward -8.965330365949827 Loss 7.351614952087402\n",
      "Episode 19 Step 3935 Action [[0.60041435]] Reward -7.9825611317223935 Loss 0.40389448404312134\n",
      "Episode 19 Step 3936 Action [[0.57924148]] Reward -7.077205411712402 Loss 0.6174197196960449\n",
      "Episode 19 Step 3937 Action [[0.43971625]] Reward -6.265156924799343 Loss 0.6323213577270508\n",
      "Episode 19 Step 3938 Action [[0.59774829]] Reward -5.575220938312409 Loss 0.2157137244939804\n",
      "Episode 19 Step 3939 Action [[0.35959334]] Reward -5.044174256198692 Loss 0.44112032651901245\n",
      "Episode 19 Step 3940 Action [[0.80662843]] Reward -4.676496538027733 Loss 0.5783100724220276\n",
      "Episode 19 Step 3941 Action [[1.04570655]] Reward -4.488018431385321 Loss 0.09166551381349564\n",
      "Episode 19 Step 3942 Action [[1.14235857]] Reward -4.4550682049906305 Loss 0.4835249185562134\n",
      "Episode 19 Step 3943 Action [[1.08551202]] Reward -4.567889535928949 Loss 0.454293429851532\n",
      "Episode 19 Step 3944 Action [[1.29406516]] Reward -4.829098218938952 Loss 0.16821834444999695\n",
      "Episode 19 Step 3945 Action [[1.02270986]] Reward -5.218493280452586 Loss 0.08276219666004181\n",
      "Episode 19 Step 3946 Action [[1.09682338]] Reward -5.757427992689451 Loss 0.6690762042999268\n",
      "Episode 19 Step 3947 Action [[1.19990403]] Reward -6.417568021822438 Loss 0.24764762818813324\n",
      "Episode 19 Step 3948 Action [[0.97682238]] Reward -7.169376506138248 Loss 0.17349952459335327\n",
      "Episode 19 Step 3949 Action [[0.99005939]] Reward -8.01917296871084 Loss 0.12475427985191345\n",
      "Episode 19 Step 3950 Action [[1.31858326]] Reward -8.916360200094859 Loss 0.18886932730674744\n",
      "Episode 19 Step 3951 Action [[1.19460376]] Reward -9.777414569293711 Loss 0.122573122382164\n",
      "Episode 19 Step 3952 Action [[1.50029136]] Reward -10.611155894139747 Loss 0.09725894778966904\n",
      "Episode 19 Step 3953 Action [[1.38306596]] Reward -9.820582564235611 Loss 0.14528320729732513\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 19 Step 3954 Action [[1.45198309]] Reward -8.982953938679689 Loss 0.15087801218032837\n",
      "Episode 19 Step 3955 Action [[0.20652096]] Reward -8.259864412120598 Loss 0.0527154803276062\n",
      "Episode 19 Step 3956 Action [[-0.706173]] Reward -7.701732923791864 Loss 0.06054822728037834\n",
      "Episode 19 Step 3957 Action [[-1.02338801]] Reward -7.2475302851066905 Loss 0.08531460911035538\n",
      "Episode 19 Step 3958 Action [[-0.85616247]] Reward -6.862007324758284 Loss 4.74368143081665\n",
      "Episode 19 Step 3959 Action [[-0.84865481]] Reward -6.54626932116387 Loss 0.2265120893716812\n",
      "Episode 19 Step 3960 Action [[-1.00406595]] Reward -6.31960208041003 Loss 10.92807674407959\n",
      "Episode 19 Step 3961 Action [[-1.17666544]] Reward -6.186641486244026 Loss 0.35170894861221313\n",
      "Episode 19 Step 3962 Action [[-1.2732364]] Reward -6.141869742319564 Loss 0.3696313500404358\n",
      "Episode 19 Step 3963 Action [[-1.34397106]] Reward -6.179904001773876 Loss 0.14678990840911865\n",
      "Episode 19 Step 3964 Action [[-1.34535759]] Reward -6.2953445720369015 Loss 0.2273235023021698\n",
      "Episode 19 Step 3965 Action [[-1.25577876]] Reward -6.484596827402724 Loss 0.5262994170188904\n",
      "Episode 19 Step 3966 Action [[-1.03246375]] Reward -6.747593974668901 Loss 0.19438937306404114\n",
      "Episode 19 Step 3967 Action [[-1.00803389]] Reward -7.091548994940272 Loss 0.11654829233884811\n",
      "Episode 19 Step 3968 Action [[-1.24342588]] Reward -7.502149644431386 Loss 0.1707807183265686\n",
      "Episode 19 Step 3969 Action [[-1.21539317]] Reward -7.941893535540732 Loss 0.23414170742034912\n",
      "Episode 19 Step 3970 Action [[-0.53233044]] Reward -8.403862931467222 Loss 0.15496613085269928\n",
      "Episode 19 Step 3971 Action [[0.09571191]] Reward -8.934326892074278 Loss 0.07320782542228699\n",
      "Episode 19 Step 3972 Action [[0.38293474]] Reward -9.544448423329005 Loss 0.18186378479003906\n",
      "Episode 19 Step 3973 Action [[0.81766941]] Reward -10.21051947317936 Loss 0.4147598147392273\n",
      "Episode 19 Step 3974 Action [[0.92067265]] Reward -9.717326068668598 Loss 0.18782642483711243\n",
      "Episode 19 Step 3975 Action [[0.7353284]] Reward -9.092965213518236 Loss 13.60413932800293\n",
      "Episode 19 Step 3976 Action [[0.70255017]] Reward -8.458109360373285 Loss 0.8110604286193848\n",
      "Episode 19 Step 3977 Action [[0.84781416]] Reward -7.84714954392957 Loss 1.4467215538024902\n",
      "Episode 19 Step 3978 Action [[1.05387459]] Reward -7.28748738373209 Loss 2.890049457550049\n",
      "Episode 19 Step 3979 Action [[1.5900951]] Reward -6.793951696304441 Loss 0.21170620620250702\n",
      "Episode 19 Step 3980 Action [[1.65071244]] Reward -6.372906120523401 Loss 14.668091773986816\n",
      "Episode 19 Step 3981 Action [[1.64570212]] Reward -6.010865729965384 Loss 0.19521769881248474\n",
      "Episode 19 Step 3982 Action [[1.85275633]] Reward -5.717706436364883 Loss 0.3158039450645447\n",
      "Episode 19 Step 3983 Action [[1.44370239]] Reward -5.49740546256258 Loss 0.41820183396339417\n",
      "Episode 19 Step 3984 Action [[1.59370062]] Reward -5.361374710008692 Loss 0.25088316202163696\n",
      "Episode 19 Step 3985 Action [[1.25665873]] Reward -5.318666010026244 Loss 0.28343045711517334\n",
      "Episode 19 Step 3986 Action [[1.52421999]] Reward -5.384983404506202 Loss 10.415044784545898\n",
      "Episode 19 Step 3987 Action [[1.21679604]] Reward -5.5466469855253075 Loss 0.20400720834732056\n",
      "Episode 19 Step 3988 Action [[1.28752197]] Reward -5.821407211189616 Loss 0.310808002948761\n",
      "Episode 19 Step 3989 Action [[1.07352085]] Reward -6.193145904920511 Loss 7.731812953948975\n",
      "Episode 19 Step 3990 Action [[1.34190204]] Reward -6.672069508384776 Loss 0.9686437845230103\n",
      "Episode 19 Step 3991 Action [[1.39277728]] Reward -7.209687857880704 Loss 0.8495219945907593\n",
      "Episode 19 Step 3992 Action [[1.57103148]] Reward -7.792406232431447 Loss 0.193354070186615\n",
      "Episode 19 Step 3993 Action [[1.28108122]] Reward -8.37973214002982 Loss 0.16924747824668884\n",
      "Episode 19 Step 3994 Action [[1.08378439]] Reward -8.990033119912164 Loss 0.43730825185775757\n",
      "Episode 19 Step 3995 Action [[1.04069331]] Reward -9.602908246393424 Loss 0.4186810851097107\n",
      "Episode 19 Step 3996 Action [[0.82407409]] Reward -10.185906444490938 Loss 0.3890407085418701\n",
      "Episode 19 Step 3997 Action [[0.91786016]] Reward -9.719934893379628 Loss 10.036303520202637\n",
      "Episode 19 Step 3998 Action [[1.04006272]] Reward -9.143409794300974 Loss 0.973767101764679\n",
      "Episode 19 Step 3999 Action [[0.76236379]] Reward -8.645782592147793 Loss 0.6453270316123962\n",
      "TOTAL REWARD @ 19-th Episode  : Reward -1376.8463955060745\n",
      "Total Step: 4000\n",
      "\n",
      "Episode : 20 Replay Buffer 4000\n",
      "[ 0.62599137 -0.77982998  0.42045632]\n",
      "Episode 20 Step 4000 Action [[0.88516115]] Reward -0.8184027975702745 Loss 0.4183320999145508\n",
      "Episode 20 Step 4001 Action [[0.97235636]] Reward -0.8038190788449285 Loss 0.5580510497093201\n",
      "Episode 20 Step 4002 Action [[0.84777314]] Reward -0.8685064358416511 Loss 3.5652170181274414\n",
      "Episode 20 Step 4003 Action [[0.97767603]] Reward -1.0237363167081799 Loss 0.9605671167373657\n",
      "Episode 20 Step 4004 Action [[1.07194136]] Reward -1.2759990717691256 Loss 0.2888193726539612\n",
      "Episode 20 Step 4005 Action [[1.53869228]] Reward -1.643262618407572 Loss 0.16501891613006592\n",
      "Episode 20 Step 4006 Action [[1.5604093]] Reward -2.112473511173749 Loss 0.8632463216781616\n",
      "Episode 20 Step 4007 Action [[1.51101928]] Reward -2.7314010278361347 Loss 0.3605424761772156\n",
      "Episode 20 Step 4008 Action [[1.11328432]] Reward -3.5361134229453106 Loss 0.16257314383983612\n",
      "Episode 20 Step 4009 Action [[1.00328123]] Reward -4.606458822781909 Loss 4.18036413192749\n",
      "Episode 20 Step 4010 Action [[1.10026239]] Reward -5.940918937761054 Loss 0.18104231357574463\n",
      "Episode 20 Step 4011 Action [[0.92687106]] Reward -7.497008445940495 Loss 0.15299786627292633\n",
      "Episode 20 Step 4012 Action [[1.00992115]] Reward -9.271333730640889 Loss 0.1318701058626175\n",
      "Episode 20 Step 4013 Action [[0.97194152]] Reward -11.133187411096973 Loss 0.33378487825393677\n",
      "Episode 20 Step 4014 Action [[0.82902384]] Reward -12.992904563418787 Loss 0.15066346526145935\n",
      "Episode 20 Step 4015 Action [[0.84732085]] Reward -11.817721228168041 Loss 0.17014722526073456\n",
      "Episode 20 Step 4016 Action [[0.87376006]] Reward -9.952897336762602 Loss 0.09741692245006561\n",
      "Episode 20 Step 4017 Action [[-0.41353621]] Reward -8.175265874487057 Loss 0.24862030148506165\n",
      "Episode 20 Step 4018 Action [[-0.49988214]] Reward -6.708908608584303 Loss 0.3123299181461334\n",
      "Episode 20 Step 4019 Action [[-0.41496943]] Reward -5.416292888050498 Loss 7.03278112411499\n",
      "Episode 20 Step 4020 Action [[-0.59478271]] Reward -4.3207166648101 Loss 0.2752101421356201\n",
      "Episode 20 Step 4021 Action [[-0.906366]] Reward -3.46190340950864 Loss 0.5305405855178833\n",
      "Episode 20 Step 4022 Action [[-0.94368282]] Reward -2.829422236781564 Loss 0.2241482436656952\n",
      "Episode 20 Step 4023 Action [[-0.87473213]] Reward -2.3833233046587416 Loss 0.054062504321336746\n",
      "Episode 20 Step 4024 Action [[-0.72387813]] Reward -2.1078023975873807 Loss 0.24383723735809326\n",
      "Episode 20 Step 4025 Action [[-0.35259924]] Reward -1.9991046164146167 Loss 0.3684556484222412\n",
      "Episode 20 Step 4026 Action [[-0.3644703]] Reward -2.0731538398671607 Loss 0.17233802378177643\n",
      "Episode 20 Step 4027 Action [[-0.57182404]] Reward -2.344353562225123 Loss 0.07174622267484665\n",
      "Episode 20 Step 4028 Action [[-0.84907727]] Reward -2.810620232447516 Loss 0.1563396453857422\n",
      "Episode 20 Step 4029 Action [[-1.25694802]] Reward -3.4666608656683855 Loss 0.22373776137828827\n",
      "Episode 20 Step 4030 Action [[-1.15093894]] Reward -4.289545021116232 Loss 0.11624933779239655\n",
      "Episode 20 Step 4031 Action [[-1.09456459]] Reward -5.3238806802931675 Loss 0.05644277110695839\n",
      "Episode 20 Step 4032 Action [[-0.8749606]] Reward -6.561701788920374 Loss 0.1179688572883606\n",
      "Episode 20 Step 4033 Action [[-0.768617]] Reward -8.004452596295698 Loss 0.258613258600235\n",
      "Episode 20 Step 4034 Action [[-0.81663431]] Reward -9.590628678615621 Loss 0.10807418823242188\n",
      "Episode 20 Step 4035 Action [[-0.81720411]] Reward -11.217003737607728 Loss 0.06884190440177917\n",
      "Episode 20 Step 4036 Action [[-0.79333704]] Reward -12.105651534627425 Loss 0.2531315088272095\n",
      "Episode 20 Step 4037 Action [[0.42837007]] Reward -10.486942217698584 Loss 4.002383708953857\n",
      "Episode 20 Step 4038 Action [[0.78041197]] Reward -9.038476480197678 Loss 0.13167208433151245\n",
      "Episode 20 Step 4039 Action [[0.75297817]] Reward -7.671612502897876 Loss 0.11294238269329071\n",
      "Episode 20 Step 4040 Action [[0.77445795]] Reward -6.393434019746394 Loss 0.23345109820365906\n",
      "Episode 20 Step 4041 Action [[0.8899306]] Reward -5.261665665415952 Loss 2.778409242630005\n",
      "Episode 20 Step 4042 Action [[0.87393347]] Reward -4.3152423315779 Loss 0.160830557346344\n",
      "Episode 20 Step 4043 Action [[0.8293677]] Reward -3.556011741107477 Loss 0.14201757311820984\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 20 Step 4044 Action [[0.7443808]] Reward -2.9866294652895564 Loss 0.27989503741264343\n",
      "Episode 20 Step 4045 Action [[0.92249207]] Reward -2.604758587447858 Loss 0.18111154437065125\n",
      "Episode 20 Step 4046 Action [[1.11683377]] Reward -2.4073836697189375 Loss 0.11444354802370071\n",
      "Episode 20 Step 4047 Action [[1.1254938]] Reward -2.3748369423241438 Loss 0.0661630630493164\n",
      "Episode 20 Step 4048 Action [[1.2490563]] Reward -2.5005306040201707 Loss 0.16508525609970093\n",
      "Episode 20 Step 4049 Action [[1.3804979]] Reward -2.782023604027365 Loss 0.12415372580289841\n",
      "Episode 20 Step 4050 Action [[1.0868771]] Reward -3.217241564921274 Loss 0.1231052502989769\n",
      "Episode 20 Step 4051 Action [[1.0417306]] Reward -3.848017628564666 Loss 0.1856713891029358\n",
      "Episode 20 Step 4052 Action [[1.00070915]] Reward -4.677315074455191 Loss 0.18032707273960114\n",
      "Episode 20 Step 4053 Action [[0.91283635]] Reward -5.707789787251012 Loss 0.06192031502723694\n",
      "Episode 20 Step 4054 Action [[1.15103054]] Reward -6.934380091875269 Loss 0.0532335489988327\n",
      "Episode 20 Step 4055 Action [[0.98835763]] Reward -8.270827254779384 Loss 0.16699817776679993\n",
      "Episode 20 Step 4056 Action [[0.93265949]] Reward -9.712772291176734 Loss 0.08329334855079651\n",
      "Episode 20 Step 4057 Action [[1.08257074]] Reward -11.178866527763507 Loss 0.08288340270519257\n",
      "Episode 20 Step 4058 Action [[0.95994913]] Reward -11.428335828104467 Loss 4.981929302215576\n",
      "Episode 20 Step 4059 Action [[1.19985572]] Reward -9.94820579600495 Loss 0.14621219038963318\n",
      "Episode 20 Step 4060 Action [[1.24413619]] Reward -8.5193475645605 Loss 2.6297311782836914\n",
      "Episode 20 Step 4061 Action [[-0.56447434]] Reward -7.236384470316954 Loss 0.1703660488128662\n",
      "Episode 20 Step 4062 Action [[-0.70150026]] Reward -6.260930689504004 Loss 4.950177192687988\n",
      "Episode 20 Step 4063 Action [[-0.55838111]] Reward -5.425954875326645 Loss 2.427900791168213\n",
      "Episode 20 Step 4064 Action [[-0.58152172]] Reward -4.740224481131378 Loss 0.23098480701446533\n",
      "Episode 20 Step 4065 Action [[-0.35493534]] Reward -4.233669927090651 Loss 0.2680746614933014\n",
      "Episode 20 Step 4066 Action [[0.14303459]] Reward -3.9144639871488938 Loss 0.3168330788612366\n",
      "Episode 20 Step 4067 Action [[-0.14778654]] Reward -3.8129652573104376 Loss 15.074577331542969\n",
      "Episode 20 Step 4068 Action [[-0.3569392]] Reward -3.9417312436101763 Loss 0.630079448223114\n",
      "Episode 20 Step 4069 Action [[-0.57578844]] Reward -4.281938898075153 Loss 0.9025737047195435\n",
      "Episode 20 Step 4070 Action [[-0.37536731]] Reward -4.8141255935173515 Loss 3.402853012084961\n",
      "Episode 20 Step 4071 Action [[-0.63852642]] Reward -5.558813253047833 Loss 0.3221169114112854\n",
      "Episode 20 Step 4072 Action [[-0.65868734]] Reward -6.463456814089891 Loss 0.6638235449790955\n",
      "Episode 20 Step 4073 Action [[-0.4960781]] Reward -7.513449653858871 Loss 0.372329980134964\n",
      "Episode 20 Step 4074 Action [[-0.68067799]] Reward -8.695136615109096 Loss 0.15984441339969635\n",
      "Episode 20 Step 4075 Action [[-0.59535686]] Reward -9.912256525782919 Loss 0.2535419762134552\n",
      "Episode 20 Step 4076 Action [[0.32777348]] Reward -11.139024573425344 Loss 0.23353934288024902\n",
      "Episode 20 Step 4077 Action [[0.49574446]] Reward -10.443909781238577 Loss 0.3356950879096985\n",
      "Episode 20 Step 4078 Action [[0.63888389]] Reward -9.283271290196367 Loss 0.38194897770881653\n",
      "Episode 20 Step 4079 Action [[1.06566821]] Reward -8.151727824961101 Loss 0.3033173680305481\n",
      "Episode 20 Step 4080 Action [[0.95601671]] Reward -7.105596340859917 Loss 0.18969501554965973\n",
      "Episode 20 Step 4081 Action [[0.78923688]] Reward -6.131337662865813 Loss 0.25322425365448\n",
      "Episode 20 Step 4082 Action [[1.0473109]] Reward -5.269893698390586 Loss 0.2573280930519104\n",
      "Episode 20 Step 4083 Action [[1.25085222]] Reward -4.573630011948715 Loss 0.22456365823745728\n",
      "Episode 20 Step 4084 Action [[1.19321452]] Reward -4.03760828744223 Loss 0.1422758400440216\n",
      "Episode 20 Step 4085 Action [[1.06505481]] Reward -3.6528331153966223 Loss 0.196285218000412\n",
      "Episode 20 Step 4086 Action [[1.10212848]] Reward -3.427359698483295 Loss 0.19944313168525696\n",
      "Episode 20 Step 4087 Action [[1.01911108]] Reward -3.367227192268757 Loss 14.806047439575195\n",
      "Episode 20 Step 4088 Action [[1.07225202]] Reward -3.4762338531205583 Loss 10.176648139953613\n",
      "Episode 20 Step 4089 Action [[0.99473708]] Reward -3.754551554787517 Loss 0.6599644422531128\n",
      "Episode 20 Step 4090 Action [[1.08722011]] Reward -4.211063470809458 Loss 4.386836051940918\n",
      "Episode 20 Step 4091 Action [[1.252389]] Reward -4.836124430231587 Loss 1.613149881362915\n",
      "Episode 20 Step 4092 Action [[1.53901805]] Reward -5.608905739671397 Loss 0.4710802733898163\n",
      "Episode 20 Step 4093 Action [[1.36439298]] Reward -6.484612102190919 Loss 0.6298071146011353\n",
      "Episode 20 Step 4094 Action [[1.58281402]] Reward -7.480826478748408 Loss 1.6751043796539307\n",
      "Episode 20 Step 4095 Action [[1.35726693]] Reward -8.511531042337026 Loss 0.3060643672943115\n",
      "Episode 20 Step 4096 Action [[1.57688212]] Reward -9.586633832122669 Loss 5.059645652770996\n",
      "Episode 20 Step 4097 Action [[1.51319555]] Reward -10.599135010954962 Loss 1.1985172033309937\n",
      "Episode 20 Step 4098 Action [[1.48765616]] Reward -10.464533533972638 Loss 5.556867599487305\n",
      "Episode 20 Step 4099 Action [[1.38706423]] Reward -9.367426768824197 Loss 1.1336331367492676\n",
      "Episode 20 Step 4100 Action [[0.90589305]] Reward -8.38570717775644 Loss 0.3661479949951172\n",
      "Episode 20 Step 4101 Action [[0.14644956]] Reward -7.567268217510518 Loss 1.033379077911377\n",
      "Episode 20 Step 4102 Action [[-0.79186988]] Reward -6.9181302895858074 Loss 0.8789134621620178\n",
      "Episode 20 Step 4103 Action [[-0.9692712]] Reward -6.411676507703902 Loss 0.37905511260032654\n",
      "Episode 20 Step 4104 Action [[-0.87567318]] Reward -6.004806666776933 Loss 0.4875795245170593\n",
      "Episode 20 Step 4105 Action [[-0.90357047]] Reward -5.703064228054169 Loss 1.2198777198791504\n",
      "Episode 20 Step 4106 Action [[-0.76725388]] Reward -5.520218693890533 Loss 0.5976048707962036\n",
      "Episode 20 Step 4107 Action [[-0.69826965]] Reward -5.466256002747049 Loss 0.20175231993198395\n",
      "Episode 20 Step 4108 Action [[-0.8604215]] Reward -5.551169769041328 Loss 0.44093430042266846\n",
      "Episode 20 Step 4109 Action [[-0.8224552]] Reward -5.765141255389467 Loss 0.6568822860717773\n",
      "Episode 20 Step 4110 Action [[-0.79745302]] Reward -6.1057753389389955 Loss 0.4845392107963562\n",
      "Episode 20 Step 4111 Action [[-0.62828764]] Reward -6.565115662925957 Loss 0.3011210262775421\n",
      "Episode 20 Step 4112 Action [[-0.83842413]] Reward -7.144484637404149 Loss 0.39414918422698975\n",
      "Episode 20 Step 4113 Action [[-0.87881828]] Reward -7.794973195104789 Loss 0.7198785543441772\n",
      "Episode 20 Step 4114 Action [[-0.63197424]] Reward -8.495696023022786 Loss 0.5036765336990356\n",
      "Episode 20 Step 4115 Action [[-0.13218574]] Reward -9.248650746824788 Loss 14.289714813232422\n",
      "Episode 20 Step 4116 Action [[0.54513486]] Reward -10.06739417957587 Loss 0.15578693151474\n",
      "Episode 20 Step 4117 Action [[1.03902879]] Reward -10.183985004635662 Loss 0.20409677922725677\n",
      "Episode 20 Step 4118 Action [[1.00048045]] Reward -9.412615108605937 Loss 0.21818669140338898\n",
      "Episode 20 Step 4119 Action [[1.00573709]] Reward -8.620625465375667 Loss 0.10883018374443054\n",
      "Episode 20 Step 4120 Action [[1.39863702]] Reward -7.840197731053755 Loss 11.305482864379883\n",
      "Episode 20 Step 4121 Action [[1.42058946]] Reward -7.112772102142775 Loss 0.319133996963501\n",
      "Episode 20 Step 4122 Action [[1.27527425]] Reward -6.434007862715194 Loss 0.3997877836227417\n",
      "Episode 20 Step 4123 Action [[1.35078676]] Reward -5.823535712074917 Loss 0.35671180486679077\n",
      "Episode 20 Step 4124 Action [[1.58158578]] Reward -5.313873344629071 Loss 0.12717370688915253\n",
      "Episode 20 Step 4125 Action [[1.80480112]] Reward -4.915978730554468 Loss 4.794684410095215\n",
      "Episode 20 Step 4126 Action [[1.9727775]] Reward -4.625280702168289 Loss 0.21238009631633759\n",
      "Episode 20 Step 4127 Action [[1.69572319]] Reward -4.433473926809759 Loss 14.989799499511719\n",
      "Episode 20 Step 4128 Action [[1.46300901]] Reward -4.346164476074907 Loss 0.8302379846572876\n",
      "Episode 20 Step 4129 Action [[1.53982554]] Reward -4.383048645596451 Loss 1.1150838136672974\n",
      "Episode 20 Step 4130 Action [[1.13868581]] Reward -4.543272060972908 Loss 0.4278324246406555\n",
      "Episode 20 Step 4131 Action [[1.1375422]] Reward -4.853070070565857 Loss 0.5020337700843811\n",
      "Episode 20 Step 4132 Action [[1.0401503]] Reward -5.305489639795553 Loss 0.7148650884628296\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 20 Step 4133 Action [[1.12721282]] Reward -5.901737385804918 Loss 0.6356731653213501\n",
      "Episode 20 Step 4134 Action [[0.84467968]] Reward -6.613203672694915 Loss 0.5638813972473145\n",
      "Episode 20 Step 4135 Action [[1.03591921]] Reward -7.455952109345777 Loss 13.440845489501953\n",
      "Episode 20 Step 4136 Action [[1.5429262]] Reward -8.357126592621594 Loss 0.9275873899459839\n",
      "Episode 20 Step 4137 Action [[1.80863356]] Reward -9.223093376091242 Loss 0.8746241927146912\n",
      "Episode 20 Step 4138 Action [[1.7316469]] Reward -10.021600444260498 Loss 0.7831855416297913\n",
      "Episode 20 Step 4139 Action [[1.7082435]] Reward -10.432974122566442 Loss 0.48458343744277954\n",
      "Episode 20 Step 4140 Action [[1.59503565]] Reward -9.55612669764834 Loss 0.23855194449424744\n",
      "Episode 20 Step 4141 Action [[1.62865722]] Reward -8.793821382284802 Loss 0.7862770557403564\n",
      "Episode 20 Step 4142 Action [[1.33840853]] Reward -8.169228668144681 Loss 6.797825813293457\n",
      "Episode 20 Step 4143 Action [[0.22633373]] Reward -7.712056745515669 Loss 0.22222092747688293\n",
      "Episode 20 Step 4144 Action [[-0.3243253]] Reward -7.409533526548463 Loss 0.7033278942108154\n",
      "Episode 20 Step 4145 Action [[-0.73482239]] Reward -7.208752036832932 Loss 0.7294644117355347\n",
      "Episode 20 Step 4146 Action [[-0.67482753]] Reward -7.086122460558704 Loss 0.48853617906570435\n",
      "Episode 20 Step 4147 Action [[-0.82303538]] Reward -7.039322833544855 Loss 0.18770630657672882\n",
      "Episode 20 Step 4148 Action [[-1.04054464]] Reward -7.0652424296134955 Loss 0.5271424651145935\n",
      "Episode 20 Step 4149 Action [[-1.09748638]] Reward -7.150615554716654 Loss 16.29934310913086\n",
      "Episode 20 Step 4150 Action [[-1.36803447]] Reward -7.289651824166772 Loss 13.75725269317627\n",
      "Episode 20 Step 4151 Action [[-1.27590492]] Reward -7.460276031748521 Loss 2.7022969722747803\n",
      "Episode 20 Step 4152 Action [[-1.18662283]] Reward -7.665524282258285 Loss 11.811089515686035\n",
      "Episode 20 Step 4153 Action [[-1.57750087]] Reward -7.903710824353008 Loss 1.6293818950653076\n",
      "Episode 20 Step 4154 Action [[-1.26465186]] Reward -8.134576966302852 Loss 2.333191394805908\n",
      "Episode 20 Step 4155 Action [[-0.805856]] Reward -8.381236623099513 Loss 0.8700666427612305\n",
      "Episode 20 Step 4156 Action [[-0.36142786]] Reward -8.659864015419576 Loss 8.606555938720703\n",
      "Episode 20 Step 4157 Action [[-0.24079699]] Reward -8.98193998306845 Loss 1.0997662544250488\n",
      "Episode 20 Step 4158 Action [[0.29323275]] Reward -9.332907438268974 Loss 0.5897085666656494\n",
      "Episode 20 Step 4159 Action [[0.23958525]] Reward -9.739152013423828 Loss 0.9689815640449524\n",
      "Episode 20 Step 4160 Action [[0.24596634]] Reward -9.91465392653238 Loss 15.33004379272461\n",
      "Episode 20 Step 4161 Action [[0.56081271]] Reward -9.512161632536747 Loss 2.8704328536987305\n",
      "Episode 20 Step 4162 Action [[0.86840535]] Reward -9.112184197857557 Loss 0.8668539524078369\n",
      "Episode 20 Step 4163 Action [[0.93522509]] Reward -8.715288122617988 Loss 0.47591644525527954\n",
      "Episode 20 Step 4164 Action [[0.90594019]] Reward -8.323535530242285 Loss 0.9277275800704956\n",
      "Episode 20 Step 4165 Action [[0.86005312]] Reward -7.948668666888498 Loss 2.3176331520080566\n",
      "Episode 20 Step 4166 Action [[1.05611845]] Reward -7.606535636489961 Loss 0.8128057718276978\n",
      "Episode 20 Step 4167 Action [[0.63386834]] Reward -7.308148599933393 Loss 0.5021175146102905\n",
      "Episode 20 Step 4168 Action [[0.45191838]] Reward -7.065637581229016 Loss 1.0346838235855103\n",
      "Episode 20 Step 4169 Action [[0.71218748]] Reward -6.907758745191882 Loss 0.5800136923789978\n",
      "Episode 20 Step 4170 Action [[0.53599895]] Reward -6.836835815505309 Loss 1.4792226552963257\n",
      "Episode 20 Step 4171 Action [[0.76151156]] Reward -6.8587335089940575 Loss 0.38647717237472534\n",
      "Episode 20 Step 4172 Action [[0.75608519]] Reward -6.963761042924956 Loss 0.4934181272983551\n",
      "Episode 20 Step 4173 Action [[1.06622366]] Reward -7.148630121380261 Loss 0.6464356184005737\n",
      "Episode 20 Step 4174 Action [[1.13342412]] Reward -7.385891583097276 Loss 0.39482009410858154\n",
      "Episode 20 Step 4175 Action [[1.22342487]] Reward -7.666302370928794 Loss 0.34611424803733826\n",
      "Episode 20 Step 4176 Action [[1.27378003]] Reward -7.973886728270132 Loss 0.6862558722496033\n",
      "Episode 20 Step 4177 Action [[1.35458304]] Reward -8.295471412364883 Loss 0.6899158954620361\n",
      "Episode 20 Step 4178 Action [[1.46373652]] Reward -8.613992637454796 Loss 0.3684350848197937\n",
      "Episode 20 Step 4179 Action [[1.28809411]] Reward -8.910729894814235 Loss 0.26131895184516907\n",
      "Episode 20 Step 4180 Action [[1.35851813]] Reward -9.192877293039544 Loss 4.075674533843994\n",
      "Episode 20 Step 4181 Action [[1.61832745]] Reward -9.439231773111686 Loss 0.23625105619430542\n",
      "Episode 20 Step 4182 Action [[1.32365133]] Reward -9.62357850802074 Loss 0.23254097998142242\n",
      "Episode 20 Step 4183 Action [[1.32594845]] Reward -9.768756307761212 Loss 0.2634362280368805\n",
      "Episode 20 Step 4184 Action [[1.54154747]] Reward -9.862236398929817 Loss 0.7086588740348816\n",
      "Episode 20 Step 4185 Action [[1.43220246]] Reward -9.857395330222554 Loss 0.3648572564125061\n",
      "Episode 20 Step 4186 Action [[1.61282487]] Reward -9.858353077067463 Loss 0.25911009311676025\n",
      "Episode 20 Step 4187 Action [[1.65318125]] Reward -9.763333377126498 Loss 0.18923687934875488\n",
      "Episode 20 Step 4188 Action [[1.7343677]] Reward -9.607406162391742 Loss 0.37400782108306885\n",
      "Episode 20 Step 4189 Action [[1.6198092]] Reward -9.393241127341916 Loss 0.22227677702903748\n",
      "Episode 20 Step 4190 Action [[1.60292742]] Reward -9.129883704972412 Loss 0.1316428929567337\n",
      "Episode 20 Step 4191 Action [[1.42390663]] Reward -8.827349294192986 Loss 0.19880685210227966\n",
      "Episode 20 Step 4192 Action [[1.27679761]] Reward -8.498005200545485 Loss 0.3726133108139038\n",
      "Episode 20 Step 4193 Action [[1.43753762]] Reward -8.160020823218717 Loss 0.2547929883003235\n",
      "Episode 20 Step 4194 Action [[1.2916117]] Reward -7.827520128747841 Loss 0.2100500464439392\n",
      "Episode 20 Step 4195 Action [[1.33982123]] Reward -7.508980947387486 Loss 3.3589112758636475\n",
      "Episode 20 Step 4196 Action [[1.20882637]] Reward -7.219629557531504 Loss 0.18356353044509888\n",
      "Episode 20 Step 4197 Action [[1.46050738]] Reward -6.97149709157929 Loss 0.10982871055603027\n",
      "Episode 20 Step 4198 Action [[1.28262779]] Reward -6.771065455410423 Loss 0.10677885264158249\n",
      "Episode 20 Step 4199 Action [[1.39257425]] Reward -6.624282378271256 Loss 0.11945062130689621\n",
      "TOTAL REWARD @ 20-th Episode  : Reward -1355.3914050263843\n",
      "Total Step: 4200\n",
      "\n",
      "Episode : 21 Replay Buffer 4200\n",
      "[ 0.61766883 -0.78643831  0.4712695 ]\n",
      "Episode 21 Step 4200 Action [[1.1188537]] Reward -0.8425248702304661 Loss 0.06445926427841187\n",
      "Episode 21 Step 4201 Action [[1.24937952]] Reward -0.816414367681975 Loss 0.08988021314144135\n",
      "Episode 21 Step 4202 Action [[1.61157138]] Reward -0.8616796584927013 Loss 0.08598006516695023\n",
      "Episode 21 Step 4203 Action [[1.53370121]] Reward -0.9653445920409417 Loss 0.09162811934947968\n",
      "Episode 21 Step 4204 Action [[1.25619628]] Reward -1.1403338694578489 Loss 0.06960482150316238\n",
      "Episode 21 Step 4205 Action [[1.22801097]] Reward -1.41859200269057 Loss 0.07678887248039246\n",
      "Episode 21 Step 4206 Action [[1.3082864]] Reward -1.8183966040923716 Loss 0.06179337948560715\n",
      "Episode 21 Step 4207 Action [[1.30884347]] Reward -2.3591617236567166 Loss 5.475497245788574\n",
      "Episode 21 Step 4208 Action [[1.20834235]] Reward -3.0759258008730868 Loss 0.06828222423791885\n",
      "Episode 21 Step 4209 Action [[1.28337555]] Reward -4.013154472660216 Loss 0.15185004472732544\n",
      "Episode 21 Step 4210 Action [[0.98996227]] Reward -5.172912794847131 Loss 0.10904809087514877\n",
      "Episode 21 Step 4211 Action [[1.03174395]] Reward -6.615842353366167 Loss 0.12601415812969208\n",
      "Episode 21 Step 4212 Action [[1.33275617]] Reward -8.273332890991233 Loss 0.13823747634887695\n",
      "Episode 21 Step 4213 Action [[1.21858523]] Reward -10.018201586239813 Loss 5.3194169998168945\n",
      "Episode 21 Step 4214 Action [[0.86689255]] Reward -11.820230125988608 Loss 0.15112504363059998\n",
      "Episode 21 Step 4215 Action [[0.95375237]] Reward -12.807238176822668 Loss 0.1575523465871811\n",
      "Episode 21 Step 4216 Action [[1.05683616]] Reward -10.94687191868924 Loss 0.1252613365650177\n",
      "Episode 21 Step 4217 Action [[0.97851488]] Reward -9.121286553371792 Loss 2.958726644515991\n",
      "Episode 21 Step 4218 Action [[-0.71893963]] Reward -7.444467678922734 Loss 0.2410612255334854\n",
      "Episode 21 Step 4219 Action [[-1.21973972]] Reward -6.1367101413558345 Loss 0.13353928923606873\n",
      "Episode 21 Step 4220 Action [[-1.04530397]] Reward -5.030137355981976 Loss 0.09696243703365326\n",
      "Episode 21 Step 4221 Action [[-1.03057827]] Reward -4.077238608853962 Loss 0.08675453066825867\n",
      "Episode 21 Step 4222 Action [[-0.93891963]] Reward -3.310973067142618 Loss 3.0109643936157227\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 21 Step 4223 Action [[-0.78932785]] Reward -2.727440441870402 Loss 0.10266397893428802\n",
      "Episode 21 Step 4224 Action [[-0.94315489]] Reward -2.322717163009396 Loss 0.154445081949234\n",
      "Episode 21 Step 4225 Action [[-1.47435121]] Reward -2.096625258623045 Loss 2.942500114440918\n",
      "Episode 21 Step 4226 Action [[-1.52648963]] Reward -2.020941548852826 Loss 0.09594522416591644\n",
      "Episode 21 Step 4227 Action [[-1.36556066]] Reward -2.071510925925775 Loss 12.356569290161133\n",
      "Episode 21 Step 4228 Action [[-1.3224518]] Reward -2.258614483854389 Loss 0.13202619552612305\n",
      "Episode 21 Step 4229 Action [[-1.43807468]] Reward -2.59470630913724 Loss 0.2466086596250534\n",
      "Episode 21 Step 4230 Action [[-1.22412543]] Reward -3.081456136184379 Loss 0.3190949559211731\n",
      "Episode 21 Step 4231 Action [[-1.17097968]] Reward -3.7585458052978984 Loss 0.13720250129699707\n",
      "Episode 21 Step 4232 Action [[-1.2022163]] Reward -4.63467102563277 Loss 0.27114003896713257\n",
      "Episode 21 Step 4233 Action [[-0.98724188]] Reward -5.703244822591481 Loss 0.3678398132324219\n",
      "Episode 21 Step 4234 Action [[-1.24752544]] Reward -6.985728905567479 Loss 0.0931699275970459\n",
      "Episode 21 Step 4235 Action [[-1.47770746]] Reward -8.377360514449977 Loss 0.060273073613643646\n",
      "Episode 21 Step 4236 Action [[-0.95608351]] Reward -9.800222898773672 Loss 0.06897133588790894\n",
      "Episode 21 Step 4237 Action [[-0.83656652]] Reward -11.309376466134959 Loss 0.2618030607700348\n",
      "Episode 21 Step 4238 Action [[0.29992484]] Reward -11.54004470612474 Loss 0.1376015990972519\n",
      "Episode 21 Step 4239 Action [[0.88925565]] Reward -10.125308787096815 Loss 0.04228559508919716\n",
      "Episode 21 Step 4240 Action [[1.09742614]] Reward -8.772890330032281 Loss 3.27047061920166\n",
      "Episode 21 Step 4241 Action [[0.98295568]] Reward -7.474578733208198 Loss 21.932584762573242\n",
      "Episode 21 Step 4242 Action [[1.16921699]] Reward -6.248483472472266 Loss 0.48570942878723145\n",
      "Episode 21 Step 4243 Action [[1.20666973]] Reward -5.172463957453566 Loss 9.22264575958252\n",
      "Episode 21 Step 4244 Action [[1.38654429]] Reward -4.257350862477475 Loss 1.492103099822998\n",
      "Episode 21 Step 4245 Action [[1.33366153]] Reward -3.5220614141678297 Loss 0.7914559841156006\n",
      "Episode 21 Step 4246 Action [[1.29067194]] Reward -2.948420942764536 Loss 0.2493753284215927\n",
      "Episode 21 Step 4247 Action [[1.25585134]] Reward -2.5343555815713685 Loss 1.0141503810882568\n",
      "Episode 21 Step 4248 Action [[1.34129896]] Reward -2.274230090588802 Loss 3.420029878616333\n",
      "Episode 21 Step 4249 Action [[1.26388592]] Reward -2.1606116889002154 Loss 0.33216071128845215\n",
      "Episode 21 Step 4250 Action [[1.28421754]] Reward -2.189242063190445 Loss 0.15273594856262207\n",
      "Episode 21 Step 4251 Action [[0.8734016]] Reward -2.3620036954744745 Loss 0.5538792610168457\n",
      "Episode 21 Step 4252 Action [[0.47178325]] Reward -2.7141548626156435 Loss 12.492833137512207\n",
      "Episode 21 Step 4253 Action [[0.98241228]] Reward -3.292568990265018 Loss 0.8116485476493835\n",
      "Episode 21 Step 4254 Action [[1.07378812]] Reward -4.047881943971804 Loss 0.5276154279708862\n",
      "Episode 21 Step 4255 Action [[0.97581769]] Reward -5.000371314828487 Loss 0.18155211210250854\n",
      "Episode 21 Step 4256 Action [[0.9318596]] Reward -6.1669458219215585 Loss 0.9280977249145508\n",
      "Episode 21 Step 4257 Action [[1.13212307]] Reward -7.5234646388799025 Loss 0.5384289026260376\n",
      "Episode 21 Step 4258 Action [[1.07746997]] Reward -8.979883410486417 Loss 0.6195579171180725\n",
      "Episode 21 Step 4259 Action [[0.84428671]] Reward -10.499027714955327 Loss 0.11527499556541443\n",
      "Episode 21 Step 4260 Action [[1.09086611]] Reward -12.039646514560722 Loss 0.261391282081604\n",
      "Episode 21 Step 4261 Action [[1.24876295]] Reward -10.914267182749787 Loss 11.938485145568848\n",
      "Episode 21 Step 4262 Action [[0.95223705]] Reward -9.35185033421522 Loss 0.7551767826080322\n",
      "Episode 21 Step 4263 Action [[0.84332532]] Reward -7.932340089735934 Loss 0.4116276502609253\n",
      "Episode 21 Step 4264 Action [[-0.25831139]] Reward -6.685304447092045 Loss 0.21445736289024353\n",
      "Episode 21 Step 4265 Action [[-0.74434658]] Reward -5.708678245559707 Loss 0.47658205032348633\n",
      "Episode 21 Step 4266 Action [[-1.0211198]] Reward -4.926600892583871 Loss 0.6473578214645386\n",
      "Episode 21 Step 4267 Action [[-1.08571656]] Reward -4.317996368492614 Loss 0.15505574643611908\n",
      "Episode 21 Step 4268 Action [[-1.0111159]] Reward -3.8709383729101288 Loss 9.250322341918945\n",
      "Episode 21 Step 4269 Action [[-0.77919879]] Reward -3.5859323121836004 Loss 0.759682297706604\n",
      "Episode 21 Step 4270 Action [[-1.01368539]] Reward -3.4748867607542695 Loss 1.2467803955078125\n",
      "Episode 21 Step 4271 Action [[-0.91574774]] Reward -3.538630013634977 Loss 1.290147066116333\n",
      "Episode 21 Step 4272 Action [[-0.94798804]] Reward -3.779487065200385 Loss 0.4527808725833893\n",
      "Episode 21 Step 4273 Action [[-0.62393943]] Reward -4.197218030391774 Loss 0.5397536754608154\n",
      "Episode 21 Step 4274 Action [[-0.57073423]] Reward -4.823353727733025 Loss 0.8503562808036804\n",
      "Episode 21 Step 4275 Action [[-0.79524287]] Reward -5.650645219440815 Loss 0.2947281301021576\n",
      "Episode 21 Step 4276 Action [[-0.39981613]] Reward -6.631624097923388 Loss 0.3990286588668823\n",
      "Episode 21 Step 4277 Action [[-0.82673574]] Reward -7.809384680666074 Loss 15.040581703186035\n",
      "Episode 21 Step 4278 Action [[-0.65304704]] Reward -9.038148315050535 Loss 2.0042102336883545\n",
      "Episode 21 Step 4279 Action [[-0.67119335]] Reward -10.330298386464237 Loss 2.4952144622802734\n",
      "Episode 21 Step 4280 Action [[-0.88604143]] Reward -11.429708726881849 Loss 0.7878164052963257\n",
      "Episode 21 Step 4281 Action [[0.04170965]] Reward -10.12917727245324 Loss 0.3381953835487366\n",
      "Episode 21 Step 4282 Action [[1.15499037]] Reward -8.947572222342187 Loss 1.6942198276519775\n",
      "Episode 21 Step 4283 Action [[1.46626779]] Reward -7.8943077624697695 Loss 1.8780770301818848\n",
      "Episode 21 Step 4284 Action [[1.31370198]] Reward -6.902956828206691 Loss 0.6050601005554199\n",
      "Episode 21 Step 4285 Action [[1.29713878]] Reward -5.969357436085132 Loss 0.49638351798057556\n",
      "Episode 21 Step 4286 Action [[1.42621079]] Reward -5.142693458145145 Loss 2.5595808029174805\n",
      "Episode 21 Step 4287 Action [[1.24244997]] Reward -4.45238915197229 Loss 1.0239567756652832\n",
      "Episode 21 Step 4288 Action [[1.45545669]] Reward -3.89742721005954 Loss 13.269318580627441\n",
      "Episode 21 Step 4289 Action [[1.47845463]] Reward -3.5001553995512347 Loss 0.7762026786804199\n",
      "Episode 21 Step 4290 Action [[1.1977568]] Reward -3.2470823518191114 Loss 0.9653239846229553\n",
      "Episode 21 Step 4291 Action [[1.42481192]] Reward -3.143995407957584 Loss 11.561955451965332\n",
      "Episode 21 Step 4292 Action [[1.29906497]] Reward -3.192062210068824 Loss 0.11104045063257217\n",
      "Episode 21 Step 4293 Action [[1.14135531]] Reward -3.3949409251617246 Loss 0.17188438773155212\n",
      "Episode 21 Step 4294 Action [[0.79050548]] Reward -3.767427667798244 Loss 0.5831922888755798\n",
      "Episode 21 Step 4295 Action [[1.0096223]] Reward -4.343699576232765 Loss 0.5008262395858765\n",
      "Episode 21 Step 4296 Action [[1.26437399]] Reward -5.091775293625507 Loss 14.66621208190918\n",
      "Episode 21 Step 4297 Action [[1.2687691]] Reward -5.980390181227498 Loss 0.7223910689353943\n",
      "Episode 21 Step 4298 Action [[1.10473262]] Reward -7.003193820430834 Loss 0.4913955330848694\n",
      "Episode 21 Step 4299 Action [[1.11031473]] Reward -8.150416171910166 Loss 0.4619573950767517\n",
      "Episode 21 Step 4300 Action [[0.8587273]] Reward -9.356850348498654 Loss 0.5486572980880737\n",
      "Episode 21 Step 4301 Action [[0.93985059]] Reward -10.608204758904904 Loss 0.5678313970565796\n",
      "Episode 21 Step 4302 Action [[1.01471897]] Reward -11.062513848973348 Loss 4.3473334312438965\n",
      "Episode 21 Step 4303 Action [[0.98337222]] Reward -9.789886773953649 Loss 0.1962871551513672\n",
      "Episode 21 Step 4304 Action [[0.71290697]] Reward -8.591133190722855 Loss 3.269279956817627\n",
      "Episode 21 Step 4305 Action [[-0.18182736]] Reward -7.528960800889492 Loss 0.45891207456588745\n",
      "Episode 21 Step 4306 Action [[-0.20866008]] Reward -6.656140022433893 Loss 0.538501501083374\n",
      "Episode 21 Step 4307 Action [[-0.04219925]] Reward -5.917616306956273 Loss 0.6172725558280945\n",
      "Episode 21 Step 4308 Action [[-0.17228767]] Reward -5.335206798361875 Loss 0.5388892889022827\n",
      "Episode 21 Step 4309 Action [[-0.39415819]] Reward -4.940958955925874 Loss 0.6851058006286621\n",
      "Episode 21 Step 4310 Action [[-0.40629971]] Reward -4.733922292917953 Loss 0.49829915165901184\n",
      "Episode 21 Step 4311 Action [[-0.432143]] Reward -4.7091437770209 Loss 0.18766425549983978\n",
      "Episode 21 Step 4312 Action [[-0.4035343]] Reward -4.867432896495068 Loss 0.08174282312393188\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 21 Step 4313 Action [[-0.22364536]] Reward -5.209410676213735 Loss 0.18920762836933136\n",
      "Episode 21 Step 4314 Action [[-0.43051686]] Reward -5.745407827639907 Loss 0.31603407859802246\n",
      "Episode 21 Step 4315 Action [[-0.4348818]] Reward -6.439224647727711 Loss 0.44307732582092285\n",
      "Episode 21 Step 4316 Action [[-0.50012175]] Reward -7.277021972384772 Loss 0.2564762830734253\n",
      "Episode 21 Step 4317 Action [[-0.56418357]] Reward -8.22009850619341 Loss 0.11824055761098862\n",
      "Episode 21 Step 4318 Action [[0.35269519]] Reward -9.22411257065425 Loss 0.13746428489685059\n",
      "Episode 21 Step 4319 Action [[1.39835651]] Reward -10.383669963863438 Loss 0.22081883251667023\n",
      "Episode 21 Step 4320 Action [[1.28976477]] Reward -10.692137712463433 Loss 0.23176464438438416\n",
      "Episode 21 Step 4321 Action [[1.29724104]] Reward -9.669354426315964 Loss 0.2234601229429245\n",
      "Episode 21 Step 4322 Action [[1.08492444]] Reward -8.616450802549279 Loss 0.14944955706596375\n",
      "Episode 21 Step 4323 Action [[1.28379727]] Reward -7.557493490381231 Loss 8.557686805725098\n",
      "Episode 21 Step 4324 Action [[1.29896895]] Reward -6.5704873983277485 Loss 0.7002884745597839\n",
      "Episode 21 Step 4325 Action [[1.64467165]] Reward -5.6731876990583405 Loss 0.2143528312444687\n",
      "Episode 21 Step 4326 Action [[1.34958764]] Reward -4.907681281622148 Loss 0.17104634642601013\n",
      "Episode 21 Step 4327 Action [[1.37313678]] Reward -4.251568659302932 Loss 0.2059803605079651\n",
      "Episode 21 Step 4328 Action [[1.70202306]] Reward -3.743482631814359 Loss 0.3055180013179779\n",
      "Episode 21 Step 4329 Action [[1.24161714]] Reward -3.385445803947227 Loss 17.045318603515625\n",
      "Episode 21 Step 4330 Action [[1.12693538]] Reward -3.162730071854822 Loss 0.1925819367170334\n",
      "Episode 21 Step 4331 Action [[1.37849812]] Reward -3.1008694053379595 Loss 0.9607819318771362\n",
      "Episode 21 Step 4332 Action [[1.21546375]] Reward -3.192664119784488 Loss 1.0689551830291748\n",
      "Episode 21 Step 4333 Action [[1.13583934]] Reward -3.4459557425018863 Loss 6.244079113006592\n",
      "Episode 21 Step 4334 Action [[1.46014878]] Reward -3.8717985493958893 Loss 0.846436083316803\n",
      "Episode 21 Step 4335 Action [[0.85420803]] Reward -4.439329536497497 Loss 0.2745099365711212\n",
      "Episode 21 Step 4336 Action [[0.85765185]] Reward -5.2252211701972975 Loss 1.2170450687408447\n",
      "Episode 21 Step 4337 Action [[0.83470029]] Reward -6.190848992429163 Loss 0.89298415184021\n",
      "Episode 21 Step 4338 Action [[0.89890904]] Reward -7.316100056997287 Loss 16.832130432128906\n",
      "Episode 21 Step 4339 Action [[1.12585089]] Reward -8.548910365273919 Loss 0.936932384967804\n",
      "Episode 21 Step 4340 Action [[1.21040429]] Reward -9.802222904062422 Loss 1.7646384239196777\n",
      "Episode 21 Step 4341 Action [[1.01250589]] Reward -11.024698175170302 Loss 1.5165300369262695\n",
      "Episode 21 Step 4342 Action [[0.7526716]] Reward -10.707362757371207 Loss 0.5178976058959961\n",
      "Episode 21 Step 4343 Action [[-0.2354719]] Reward -9.449286696133665 Loss 0.8095434904098511\n",
      "Episode 21 Step 4344 Action [[-1.3026713]] Reward -8.330535272772833 Loss 1.4662785530090332\n",
      "Episode 21 Step 4345 Action [[-0.90916913]] Reward -7.3436649464731465 Loss 1.3805367946624756\n",
      "Episode 21 Step 4346 Action [[-0.95543488]] Reward -6.38957610829129 Loss 0.2166653722524643\n",
      "Episode 21 Step 4347 Action [[-0.69950704]] Reward -5.5477595814607294 Loss 0.28654083609580994\n",
      "Episode 21 Step 4348 Action [[-0.86490597]] Reward -4.833242073415476 Loss 0.7473123669624329\n",
      "Episode 21 Step 4349 Action [[-0.81077265]] Reward -4.290504397803663 Loss 0.8228487968444824\n",
      "Episode 21 Step 4350 Action [[-0.54152502]] Reward -3.915973053888654 Loss 0.2603859305381775\n",
      "Episode 21 Step 4351 Action [[-0.65749989]] Reward -3.7223112167565136 Loss 0.3155297338962555\n",
      "Episode 21 Step 4352 Action [[-0.86795712]] Reward -3.7215509837640686 Loss 0.4423266053199768\n",
      "Episode 21 Step 4353 Action [[-0.55776208]] Reward -3.900259637307953 Loss 0.5918196439743042\n",
      "Episode 21 Step 4354 Action [[-0.68270109]] Reward -4.279538808818248 Loss 14.14901065826416\n",
      "Episode 21 Step 4355 Action [[-0.88744725]] Reward -4.847266976200719 Loss 0.2886884808540344\n",
      "Episode 21 Step 4356 Action [[-0.87230956]] Reward -5.578923212029929 Loss 7.5620317459106445\n",
      "Episode 21 Step 4357 Action [[-0.94669118]] Reward -6.47081032726256 Loss 1.7744886875152588\n",
      "Episode 21 Step 4358 Action [[-0.79917495]] Reward -7.485538025949514 Loss 0.7621373534202576\n",
      "Episode 21 Step 4359 Action [[-0.78732445]] Reward -8.611830753527771 Loss 0.11316095292568207\n",
      "Episode 21 Step 4360 Action [[-0.77003762]] Reward -9.788798174300977 Loss 0.5985745191574097\n",
      "Episode 21 Step 4361 Action [[0.24301126]] Reward -10.962137107983379 Loss 1.474879264831543\n",
      "Episode 21 Step 4362 Action [[1.20792132]] Reward -10.48344280848225 Loss 0.6217595338821411\n",
      "Episode 21 Step 4363 Action [[1.51553338]] Reward -9.407050959934235 Loss 0.21322263777256012\n",
      "Episode 21 Step 4364 Action [[1.21031048]] Reward -8.33601157231569 Loss 12.512336730957031\n",
      "Episode 21 Step 4365 Action [[1.25327104]] Reward -7.254877832770903 Loss 1.3336859941482544\n",
      "Episode 21 Step 4366 Action [[1.30451828]] Reward -6.245234637289404 Loss 2.476104259490967\n",
      "Episode 21 Step 4367 Action [[1.12037239]] Reward -5.34241194074607 Loss 9.59253978729248\n",
      "Episode 21 Step 4368 Action [[1.13868638]] Reward -4.561229305374117 Loss 6.637890815734863\n",
      "Episode 21 Step 4369 Action [[0.84559748]] Reward -3.9370617097687064 Loss 0.5431568622589111\n",
      "Episode 21 Step 4370 Action [[1.27156163]] Reward -3.4729918906029154 Loss 0.6191267967224121\n",
      "Episode 21 Step 4371 Action [[1.1336625]] Reward -3.1896951197820353 Loss 0.5264788866043091\n",
      "Episode 21 Step 4372 Action [[1.01085284]] Reward -3.0646085978962967 Loss 0.09624941647052765\n",
      "Episode 21 Step 4373 Action [[1.14748702]] Reward -3.1081759394623267 Loss 0.3407580852508545\n",
      "Episode 21 Step 4374 Action [[0.90353727]] Reward -3.3173102877818783 Loss 0.716910183429718\n",
      "Episode 21 Step 4375 Action [[0.73719386]] Reward -3.7127905621375463 Loss 0.6976707577705383\n",
      "Episode 21 Step 4376 Action [[0.89426677]] Reward -4.3126575986230975 Loss 0.33954083919525146\n",
      "Episode 21 Step 4377 Action [[1.01647323]] Reward -5.096456505177884 Loss 0.1677585393190384\n",
      "Episode 21 Step 4378 Action [[0.96354152]] Reward -6.04722760758599 Loss 0.2787761688232422\n",
      "Episode 21 Step 4379 Action [[0.99705521]] Reward -7.158486514924121 Loss 0.491209477186203\n",
      "Episode 21 Step 4380 Action [[1.10549622]] Reward -8.382285497555413 Loss 0.49841374158859253\n",
      "Episode 21 Step 4381 Action [[1.27064965]] Reward -9.6500696202584 Loss 0.30407023429870605\n",
      "Episode 21 Step 4382 Action [[1.11895283]] Reward -10.882938883265787 Loss 0.25145968794822693\n",
      "Episode 21 Step 4383 Action [[1.30514347]] Reward -10.904354484597656 Loss 0.6343386769294739\n",
      "Episode 21 Step 4384 Action [[1.47449198]] Reward -9.592996902703913 Loss 0.2355472445487976\n",
      "Episode 21 Step 4385 Action [[-0.01954306]] Reward -8.367312533871736 Loss 0.2712113857269287\n",
      "Episode 21 Step 4386 Action [[-0.71478773]] Reward -7.380781009102741 Loss 0.09425398707389832\n",
      "Episode 21 Step 4387 Action [[-0.73675395]] Reward -6.53819606124015 Loss 0.1240309327840805\n",
      "Episode 21 Step 4388 Action [[-0.81937474]] Reward -5.804976290698226 Loss 0.23549595475196838\n",
      "Episode 21 Step 4389 Action [[-0.77701316]] Reward -5.206001083177316 Loss 0.14124760031700134\n",
      "Episode 21 Step 4390 Action [[-0.75685022]] Reward -4.752202889203816 Loss 10.134490966796875\n",
      "Episode 21 Step 4391 Action [[-0.87339834]] Reward -4.458153225204325 Loss 0.2302049696445465\n",
      "Episode 21 Step 4392 Action [[-1.07145283]] Reward -4.329031428774518 Loss 5.810191631317139\n",
      "Episode 21 Step 4393 Action [[-1.13649672]] Reward -4.355789654072479 Loss 0.7248051166534424\n",
      "Episode 21 Step 4394 Action [[-0.7597568]] Reward -4.530705995408307 Loss 0.5714051127433777\n",
      "Episode 21 Step 4395 Action [[-1.13125223]] Reward -4.879722860847727 Loss 0.46474021673202515\n",
      "Episode 21 Step 4396 Action [[-1.1812805]] Reward -5.364308713314958 Loss 0.3347241282463074\n",
      "Episode 21 Step 4397 Action [[-1.25898593]] Reward -5.978481807131008 Loss 0.5485355854034424\n",
      "Episode 21 Step 4398 Action [[-1.39712126]] Reward -6.699550428310567 Loss 0.5568105578422546\n",
      "Episode 21 Step 4399 Action [[-1.12560215]] Reward -7.489952063567689 Loss 0.14822567999362946\n",
      "Now we save model\n",
      "TOTAL REWARD @ 21-th Episode  : Reward -1196.5823979420375\n",
      "Total Step: 4400\n",
      "\n",
      "Episode : 22 Replay Buffer 4400\n",
      "[ 0.15806883  0.9874281  -0.97550718]\n",
      "Episode 22 Step 4400 Action [[-1.42840112]] Reward -2.091120106528434 Loss 5.629609107971191\n",
      "Episode 22 Step 4401 Action [[-1.31984486]] Reward -1.952913227121232 Loss 0.5195237994194031\n",
      "Episode 22 Step 4402 Action [[-1.4231963]] Reward -1.9464420904163886 Loss 36.84519577026367\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 22 Step 4403 Action [[-1.37512656]] Reward -2.0700997345451024 Loss 1.375098466873169\n",
      "Episode 22 Step 4404 Action [[-1.53064166]] Reward -2.332538445850091 Loss 2.46690034866333\n",
      "Episode 22 Step 4405 Action [[-1.19410527]] Reward -2.7318693353157495 Loss 8.849287033081055\n",
      "Episode 22 Step 4406 Action [[-0.88567663]] Reward -3.31755590083753 Loss 1.3733320236206055\n"
     ]
    }
   ],
   "source": [
    "def playGame(train_indicator=0):    #1 means Train, 0 means simply Run\n",
    "    \n",
    "    # Generate a Torcs environment\n",
    "    env = gym.make('Pendulum-v0')\n",
    "    env.reset()\n",
    "    \n",
    "    ##########################\n",
    "    print(env.action_space.sample())\n",
    "    print(env.observation_space.sample())\n",
    "    \n",
    "    state_dim = 3\n",
    "    \n",
    "    action_dim = 1\n",
    "    action_low = env.observation_space.low\n",
    "    action_high = env.observation_space.high\n",
    "\n",
    "    ##########################\n",
    "    # Noise process\n",
    "    exploration_mu = 0.1 * np.ones(action_dim)\n",
    "    print(exploration_mu)\n",
    "    exploration_theta = 0.15\n",
    "    exploration_sigma = 0.2\n",
    "    OU = OUNoise(action_dim, exploration_mu, exploration_theta, exploration_sigma)\n",
    "\n",
    "    BUFFER_SIZE = 100000\n",
    "    BATCH_SIZE = 32\n",
    "    GAMMA = 0.99\n",
    "    TAU = 0.001     #Target Network HyperParameters\n",
    "    LRA = 0.0001    #Learning rate for Actor\n",
    "    LRC = 0.001     #Lerning rate for Critic\n",
    "\n",
    "    np.random.seed(1337)\n",
    "\n",
    "    vision = False\n",
    "\n",
    "    EXPLORE = 100000.\n",
    "    episode_count = 2000\n",
    "    max_steps = 100000\n",
    "    reward = 0\n",
    "    done = False\n",
    "    step = 0\n",
    "    epsilon = 1\n",
    "    indicator = 0\n",
    "\n",
    "    #Tensorflow GPU optimization\n",
    "    config = tf.ConfigProto()\n",
    "    config.gpu_options.allow_growth = True\n",
    "    sess = tf.Session(config=config)\n",
    "    K.set_session(sess)\n",
    "\n",
    "    actor = ActorNetwork(sess, state_dim, action_dim, BATCH_SIZE, TAU, LRA)\n",
    "    critic = CriticNetwork(sess, state_dim, action_dim, BATCH_SIZE, TAU, LRC)\n",
    "    buff = ReplayBuffer(BUFFER_SIZE)    #Create replay buffer\n",
    "\n",
    "    #Now load the weight\n",
    "    print(\"Now we load the weight\")\n",
    "    try:\n",
    "        actor.model.load_weights(\"actormodel.h5\")\n",
    "        critic.model.load_weights(\"criticmodel.h5\")\n",
    "        actor.target_model.load_weights(\"actormodel.h5\")\n",
    "        critic.target_model.load_weights(\"criticmodel.h5\")\n",
    "        print(\"Weight load successfully\")\n",
    "    except:\n",
    "        print(\"Cannot find the weight\")\n",
    "\n",
    "    print(\"TORCS Experiment Start.\")\n",
    "    for i in range(episode_count):\n",
    "\n",
    "        print(\"Episode : \" + str(i) + \" Replay Buffer \" + str(buff.count()))\n",
    "        \n",
    "        ob = env.reset()\n",
    "        \n",
    "        s_t = np.hstack(ob)\n",
    "        print(s_t)\n",
    "     \n",
    "        total_reward = 0.\n",
    "        for j in range(max_steps):\n",
    "            env.render()\n",
    "            loss = 0 \n",
    "            epsilon -= 1.0 / EXPLORE\n",
    "            a_t = np.zeros([1,action_dim])\n",
    "            noise_t = np.zeros([1,action_dim])\n",
    "            \n",
    "            a_t_original = actor.model.predict(s_t.reshape(1, s_t.shape[0]))\n",
    "            noise_t = train_indicator * max(epsilon, 0) * OU.sample().reshape(1, action_dim)\n",
    "\n",
    "            #The following code do the stochastic brake\n",
    "            #if random.random() <= 0.1:\n",
    "            #    print(\"********Now we apply the brake***********\")\n",
    "            #    noise_t[0][2] = train_indicator * max(epsilon, 0) * OU.function(a_t_original[0][2],  0.2 , 1.00, 0.10)\n",
    "\n",
    "            a_t = a_t_original + noise_t\n",
    "\n",
    "            ob, r_t, done, info = env.step(a_t[0])\n",
    "\n",
    "            s_t1 = np.hstack(ob)\n",
    "        \n",
    "            buff.add(s_t, a_t[0], r_t, s_t1, done)      #Add replay buffer\n",
    "            \n",
    "            #Do the batch update\n",
    "            batch = buff.getBatch(BATCH_SIZE)\n",
    "            states = np.asarray([e[0] for e in batch])\n",
    "            actions = np.asarray([e[1] for e in batch])\n",
    "            rewards = np.asarray([e[2] for e in batch])\n",
    "            new_states = np.asarray([e[3] for e in batch])\n",
    "            dones = np.asarray([e[4] for e in batch])\n",
    "            y_t = np.asarray([e[1] for e in batch])\n",
    "\n",
    "            target_q_values = critic.target_model.predict([new_states, actor.target_model.predict(new_states)])  \n",
    "           \n",
    "            for k in range(len(batch)):\n",
    "                if dones[k]:\n",
    "                    y_t[k] = rewards[k]\n",
    "                else:\n",
    "                    y_t[k] = rewards[k] + GAMMA*target_q_values[k]\n",
    "       \n",
    "            if (train_indicator):\n",
    "                loss += critic.model.train_on_batch([states,actions], y_t) \n",
    "                a_for_grad = actor.model.predict(states)\n",
    "                grads = critic.gradients(states, a_for_grad)\n",
    "                actor.train(states, grads)\n",
    "                actor.target_train()\n",
    "                critic.target_train()\n",
    "\n",
    "            total_reward += r_t\n",
    "            s_t = s_t1\n",
    "        \n",
    "            print(\"Episode\", i, \"Step\", step, \"Action\", a_t, \"Reward\", r_t, \"Loss\", loss)\n",
    "        \n",
    "            step += 1\n",
    "            if done:\n",
    "                break\n",
    "\n",
    "        if np.mod(i, 3) == 0:\n",
    "            if (train_indicator):\n",
    "                print(\"Now we save model\")\n",
    "                actor.model.save_weights(\"actormodel.h5\", overwrite=True)\n",
    "                with open(\"actormodel.json\", \"w\") as outfile:\n",
    "                    json.dump(actor.model.to_json(), outfile)\n",
    "\n",
    "                critic.model.save_weights(\"criticmodel.h5\", overwrite=True)\n",
    "                with open(\"criticmodel.json\", \"w\") as outfile:\n",
    "                    json.dump(critic.model.to_json(), outfile)\n",
    "\n",
    "        print(\"TOTAL REWARD @ \" + str(i) +\"-th Episode  : Reward \" + str(total_reward))\n",
    "        print(\"Total Step: \" + str(step))\n",
    "        print(\"\")\n",
    "\n",
    "    env.end()  # This is for shutting down TORCS\n",
    "    print(\"Finish.\")\n",
    "    \n",
    "playGame(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
